From 5a0f5f2aa1f5b2a9f9611af52a3fcd548f03bc06 Mon Sep 17 00:00:00 2001 From: matt rice Date: Tue, 4 Aug 2026 16:06:30 -0700 Subject: [PATCH 01/14] Initial steps towards a `lrpar::codegen` module. --- lrpar/src/lib/codegen.rs | 316 +++++++++++++++++++++++++++++++++++++ lrpar/src/lib/ctbuilder.rs | 151 +++++------------- lrpar/src/lib/mod.rs | 2 + 3 files changed, 361 insertions(+), 108 deletions(-) create mode 100644 lrpar/src/lib/codegen.rs diff --git a/lrpar/src/lib/codegen.rs b/lrpar/src/lib/codegen.rs new file mode 100644 index 000000000..265d3a0e5 --- /dev/null +++ b/lrpar/src/lib/codegen.rs @@ -0,0 +1,316 @@ +#![deny(unfulfilled_lint_expectations)] +#![expect(dead_code)] + +use std::{error::Error, fmt, marker::PhantomData, path::Path}; + +use crate::{ + LexerTypes, RecoveryKind, RustEdition, SerialisationFormat, Visibility, + ctbuilder::{ERROR, indent}, + diagnostics::{DiagnosticFormatter, SpannedDiagnosticFormatter}, +}; + +use cfgrammar::{ + Location, Span, + header::{GrmtoolsSectionParser, Header, HeaderValue}, + yacc::{YaccGrammar, YaccKind, ast::ASTWithValidityInfo}, +}; +use lrtable::{StateGraph, StateTable}; + +pub(crate) struct ParserSrcEnv<'a> { + src: &'a str, + // We store the path here so we can generate a module name from it if needed. + // But should never use it for filesystem interaction within this module. + path: &'a Path, + diagnostics: SpannedDiagnosticFormatter<'a>, + header: Header, +} + +pub(crate) struct ParserBuildEnvArgs<'a> { + /// This allows the parser to originate from from a pre-parsed AST, rather than + /// parsing a grammar definition given as source string into an AST. + ast_originated: Option<&'a ASTWithValidityInfo>, + mod_name: Option, + rust_edition: RustEdition, + visibility: Visibility, + error_on_conflicts: bool, + show_warnings: bool, + warnings_are_errors: bool, +} + +pub(crate) struct ParserBuildEnv<'a, LexerTypesT> +where + LexerTypesT: LexerTypes, + usize: num_traits::AsPrimitive, +{ + ast_validation: ASTWithValidityInfo, + recoverer: RecoveryKind, + serialisation_format: SerialisationFormat, + // Preserve the args for generating the cache. + cache_args: ParserBuildEnvArgs<'a>, + phantom_storaget: PhantomData, + mod_name: String, +} + +pub(crate) struct ParserCodegen +where + LexerTypesT: LexerTypes, + usize: num_traits::AsPrimitive, +{ + grm: YaccGrammar, + stable: StateTable, + sgraph: StateGraph, + timestamp: String, +} + +impl<'a> ParserBuildEnvArgs<'a> { + pub(crate) fn new() -> Self { + ParserBuildEnvArgs { + ast_originated: None, + mod_name: None, + visibility: Visibility::Private, + rust_edition: RustEdition::Rust2021, + error_on_conflicts: true, + show_warnings: true, + warnings_are_errors: true, + } + } + + pub(crate) fn ast_originated(mut self, ast: Option<&'a ASTWithValidityInfo>) -> Self { + self.ast_originated = ast; + self + } + + pub(crate) fn mod_name(mut self, mod_name: Option<&str>) -> Self { + self.mod_name = mod_name.map(|s| s.to_string()); + self + } + + pub(crate) fn rust_edition(mut self, rust_edition: RustEdition) -> Self { + self.rust_edition = rust_edition; + self + } + pub(crate) fn visibility(mut self, visibility: Visibility) -> Self { + self.visibility = visibility; + self + } + pub(crate) fn error_on_conflicts(mut self, error_on_conflicts: bool) -> Self { + self.error_on_conflicts = error_on_conflicts; + self + } + pub(crate) fn show_warnings(mut self, show_warnings: bool) -> Self { + self.show_warnings = show_warnings; + self + } + pub(crate) fn warnings_are_errors(mut self, warnings_are_errors: bool) -> Self { + self.warnings_are_errors = warnings_are_errors; + self + } +} + +impl<'a> ParserSrcEnv<'a> { + pub(crate) fn new_with_defaults( + src: &'a str, + path: &'a Path, + header: Header, + ) -> ParserSrcEnv<'a> { + let diagnostics = SpannedDiagnosticFormatter::new(src, path); + ParserSrcEnv { + src, + path, + header, + diagnostics, + } + } + + pub(crate) fn yacc_diag(&self) -> &SpannedDiagnosticFormatter<'a> { + &self.diagnostics + } + + pub(crate) fn header_mut(&mut self) -> &mut Header { + &mut self.header + } + + pub(crate) fn header(&self) -> &Header { + &self.header + } + + fn merge_headers(&mut self) -> Result<(), Box> { + let (parsed_header, _) = self.parse_header()?; + Ok(self.header.merge_from(parsed_header)?) + } + + fn parse_header(&self) -> Result<(Header, usize), Box> { + GrmtoolsSectionParser::new(self.src, false) + .parse() + .map_err(|es| { + let mut out = String::new(); + out.push_str(&format!( + "\n{ERROR}{}\n", + self.yacc_diag() + .file_location_msg(" parsing the `%grmtools` section", None) + )); + for e in es { + out.push_str(&indent( + " ", + &self.yacc_diag().format_error(e).to_string(), + )); + out.push('\n'); + } + ErrorString(out).into() + }) + } + + pub(crate) fn check_unused_header_keys(&self) -> Result<(), Box> { + let unused_keys = self.header.unused(); + if !unused_keys.is_empty() { + return Err(format!("Unused keys in header: {}", unused_keys.join(", ")).into()); + } + let missing_keys = self + .header + .missing() + .iter() + .map(|s| s.as_str()) + .collect::>(); + if !missing_keys.is_empty() { + Err(format!( + "Required values were missing from the header: {}", + missing_keys.join(", ") + ) + .into()) + } else { + Ok(()) + } + } + + fn extract_ast_validation( + &mut self, + from_ast: Option<&ASTWithValidityInfo>, + ) -> Result> { + self.header.mark_used(&"yacckind".to_string()); + if let Some(ast) = from_ast { + Ok(ast.clone()) + } else if let Some(yk) = self + .header + .get("yacckind") + .map(|HeaderValue(_, val)| val) + .map(YaccKind::try_from) + .transpose()? + { + Ok(ASTWithValidityInfo::new(yk, self.src)) + } else { + Err("Missing 'yacckind'".to_string())? + } + } + + fn extract_recoverer(&mut self) -> Result> { + self.header.mark_used(&"recoverer".to_string()); + let rk_val = self + .header + .get("recoverer") + .map(|HeaderValue(_, rk_val)| rk_val); + if let Some(rk_val) = rk_val { + Ok(RecoveryKind::try_from(rk_val)?) + } else { + // Fallback to the default recoverykind. + Ok(RecoveryKind::CPCTPlus) + } + } + + fn extract_serialisation_format( + &mut self, + ) -> Result> { + self.header.mark_used(&"serialisation_format".to_string()); + if let Some(ec_val) = self + .header + .get("serialisation_format") + .map(|HeaderValue(_, ec_val)| ec_val) + { + Ok(SerialisationFormat::try_from(ec_val)?) + } else { + Ok(SerialisationFormat::VariableSizedInteger) + } + } + + fn extract_mod_name(&self, args: &ParserBuildEnvArgs) -> String { + match &args.mod_name { + Some(s) => s.to_owned(), + None => { + // The user hasn't specified a module name, so we create one automatically: what we + // do is strip off all the filename extensions (note that it's likely that inp ends + // with `y.rs`, so we potentially have to strip off more than one extension) and + // then add `_y` to the end. + let mut stem = self.path.to_str().unwrap(); + loop { + let new_stem = Path::new(stem).file_stem().unwrap().to_str().unwrap(); + if stem == new_stem { + break; + } + stem = new_stem; + } + format!("{}_y", stem) + } + } + } + + pub(crate) fn build_env( + &mut self, + args: ParserBuildEnvArgs<'a>, + ) -> Result, Box> + where + LexerTypesT: LexerTypes, + usize: num_traits::AsPrimitive, + { + self.merge_headers()?; + let ast_validation = self.extract_ast_validation(args.ast_originated)?; + let recoverer = self.extract_recoverer()?; + let serialisation_format = self.extract_serialisation_format()?; + let mod_name = self.extract_mod_name(&args); + + Ok(ParserBuildEnv { + ast_validation, + cache_args: args, + recoverer, + serialisation_format, + mod_name, + phantom_storaget: PhantomData, + }) + } +} + +impl<'a, LexerTypesT> ParserBuildEnv<'a, LexerTypesT> +where + LexerTypesT: LexerTypes, + usize: num_traits::AsPrimitive, +{ + pub(crate) fn ast_validation(&self) -> &ASTWithValidityInfo { + &self.ast_validation + } + + pub(crate) fn serialisation_format(&self) -> &SerialisationFormat { + &self.serialisation_format + } + + pub(crate) fn derived_mod_name(&self) -> &str { + &self.mod_name + } + + pub(crate) fn recoverer(&self) -> RecoveryKind { + self.recoverer + } +} + +/// A string which uses `Display` for it's `Debug` impl. +struct ErrorString(String); +impl fmt::Display for ErrorString { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + let ErrorString(s) = self; + write!(f, "{}", s) + } +} +impl fmt::Debug for ErrorString { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + let ErrorString(s) = self; + write!(f, "{}", s) + } +} +impl Error for ErrorString {} diff --git a/lrpar/src/lib/ctbuilder.rs b/lrpar/src/lib/ctbuilder.rs index 830558116..240aa44ca 100644 --- a/lrpar/src/lib/ctbuilder.rs +++ b/lrpar/src/lib/ctbuilder.rs @@ -16,6 +16,7 @@ use std::{ use crate::{ LexerTypes, RTParserBuilder, RecoveryKind, + codegen::{ParserBuildEnv, ParserBuildEnvArgs, ParserSrcEnv}, diagnostics::{DiagnosticFormatter, SpannedDiagnosticFormatter}, }; @@ -24,10 +25,7 @@ use crate::unstable_api::UnstableApi; use cfgrammar::{ Location, RIdx, Span, Symbol, - header::{ - GrmtoolsSectionParser, Header, HeaderError, HeaderErrorKind, HeaderValue, Namespaced, - Setting, Value, - }, + header::{Header, HeaderError, HeaderErrorKind, HeaderValue, Namespaced, Setting, Value}, markmap::{Entry, MergeBehavior}, yacc::{YaccGrammar, YaccKind, YaccOriginalActionKind, ast::ASTWithValidityInfo}, }; @@ -48,7 +46,7 @@ const ACTIONS_KIND_HIDDEN: &str = "__GtActionsKindHidden"; const RUST_FILE_EXT: &str = "rs"; const WARNING: &str = "[Warning]"; -const ERROR: &str = "[Error]"; +pub(crate) const ERROR: &str = "[Error]"; static GENERATED_PATHS: LazyLock>> = LazyLock::new(|| Mutex::new(HashSet::new())); @@ -727,73 +725,38 @@ where } else { read_to_string(grmp).map_err(|e| format!("When reading '{}': {e}", grmp.display()))? }; - let yacc_diag = SpannedDiagnosticFormatter::new(&inc, grmp); - let parsed_header = GrmtoolsSectionParser::new(&inc, false).parse(); - if let Err(errs) = parsed_header { - let mut out = String::new(); - out.push_str(&format!( - "\n{ERROR}{}\n", - yacc_diag.file_location_msg(" parsing the `%grmtools` section", None) - )); - for e in errs { - out.push_str(&indent(" ", &yacc_diag.format_error(e).to_string())); - } - return Err(ErrorString(out).into()); - }; - let (parsed_header, _) = parsed_header.unwrap(); - header.merge_from(parsed_header)?; - self.yacckind = header - .get("yacckind") - .map(|HeaderValue(_, val)| val) - .map(YaccKind::try_from) - .transpose()?; - header.mark_used(&"yacckind".to_string()); - let ast_validation = if let Some(ast) = &self.from_ast { - ast.clone() - } else if let Some(yk) = self.yacckind { - ASTWithValidityInfo::new(yk, &inc) - } else { - Err("Missing 'yacckind'".to_string())? - }; - - header.mark_used(&"recoverer".to_string()); - let rk_val = header.get("recoverer").map(|HeaderValue(_, rk_val)| rk_val); - - if let Some(rk_val) = rk_val { - self.recoverer = Some(RecoveryKind::try_from(rk_val)?); - } else { - // Fallback to the default recoverykind. - self.recoverer = Some(RecoveryKind::CPCTPlus); - } - header.mark_used(&"serialisation_format".to_string()); - if let Some(ec_val) = header - .get("serialisation_format") - .map(|HeaderValue(_, ec_val)| ec_val) - { - self.serialisation_format = Some(SerialisationFormat::try_from(ec_val)?); - } else { - self.serialisation_format = Some(SerialisationFormat::VariableSizedInteger); - } - self.yacckind = Some(ast_validation.yacc_kind()); - let warnings = ast_validation.ast().warnings(); + let mut src_env = ParserSrcEnv::new_with_defaults(&inc, grmp, header); + let build_args = ParserBuildEnvArgs::new() + .ast_originated(self.from_ast.as_ref()) + .mod_name(self.mod_name) + .show_warnings(self.show_warnings) + .error_on_conflicts(self.error_on_conflicts) + .warnings_are_errors(self.warnings_are_errors); + let build_env = src_env.build_env::(build_args)?; + // Temporarily we update self.yacckind and self.recoverer from the build_env + // Until codegen reads these variables from the build_env directly. + self.recoverer = Some(build_env.recoverer()); + self.yacckind = Some(build_env.ast_validation().yacc_kind()); + + let warnings = build_env.ast_validation().ast().warnings(); if self.warnings_are_errors && !warnings.is_empty() { let mut out = String::new(); out.push_str(&format!( "\n{ERROR}{}\n", - yacc_diag.file_location_msg("", None) + src_env.yacc_diag().file_location_msg("", None) )); for e in warnings { out.push_str(&format!( "{}\n", - indent(" ", &yacc_diag.format_warning(e).to_string()) + indent(" ", &src_env.yacc_diag().format_warning(e).to_string()) )); } return Err(ErrorString(out).into()); } else if !warnings.is_empty() { for w in warnings { - let ws_loc = yacc_diag.file_location_msg("", None); - let ws = indent(" ", &yacc_diag.format_warning(w).to_string()); + let ws_loc = src_env.yacc_diag().file_location_msg("", None); + let ws = indent(" ", &src_env.yacc_diag().format_warning(w).to_string()); // Assume if this variable is set we are running under cargo. if std::env::var("OUT_DIR").is_ok() && self.show_warnings { for line in ws_loc.lines().chain(ws.lines()) { @@ -805,16 +768,21 @@ where } } } - let grm = match YaccGrammar::::new_from_ast_with_validity_info(&ast_validation) { + let grm = match YaccGrammar::::new_from_ast_with_validity_info( + build_env.ast_validation(), + ) { Ok(grm) => grm, Err(errs) => { let mut out = String::new(); out.push_str(&format!( "\n{ERROR}{}\n", - yacc_diag.file_location_msg("", None) + src_env.yacc_diag().file_location_msg("", None) )); for e in errs { - out.push_str(&indent(" ", &yacc_diag.format_error(e).to_string())); + out.push_str(&indent( + " ", + &src_env.yacc_diag().format_error(e).to_string(), + )); out.push('\n'); } return Err(ErrorString(out).into()); @@ -822,7 +790,7 @@ where }; #[cfg(test)] if let Some(cb) = &self.inspect_callback { - cb(self.recoverer.expect("has a default value"))?; + cb(build_env.recoverer())?; } let rule_ids = grm @@ -831,26 +799,7 @@ where .map(|(&n, &i)| (n.to_owned(), i.as_storaget())) .collect::>(); - let derived_mod_name = match self.mod_name { - Some(s) => s.to_owned(), - None => { - // The user hasn't specified a module name, so we create one automatically: what we - // do is strip off all the filename extensions (note that it's likely that inp ends - // with `y.rs`, so we potentially have to strip off more than one extension) and - // then add `_y` to the end. - let mut stem = grmp.to_str().unwrap(); - loop { - let new_stem = Path::new(stem).file_stem().unwrap().to_str().unwrap(); - if stem == new_stem { - break; - } - stem = new_stem; - } - format!("{}_y", stem) - } - }; - - let cache = self.rebuild_cache(&derived_mod_name, &grm); + let cache = self.rebuild_cache(build_env.derived_mod_name(), &grm); // We don't need to go through the full rigmarole of generating an output file if all of // the following are true: the output file exists; it is newer than the input file; and the @@ -903,9 +852,9 @@ where (None, Some(j)) if 0 == c.sr_len() && j == c.rr_len() => (), (None, None) if 0 == c.rr_len() && 0 == c.sr_len() => (), _ => { - let conflicts_diagnostic = yacc_diag.format_conflicts::( + let conflicts_diagnostic = src_env.yacc_diag().format_conflicts::( &grm, - ast_validation.ast(), + build_env.ast_validation().ast(), c, &sgraph, &stable, @@ -928,33 +877,19 @@ where } else { rt }; - inspector_rt(&mut header, rt, &rule_ids, grmp)? + inspector_rt(src_env.header_mut(), rt, &rule_ids, grmp)? } - let unused_keys = header.unused(); - if !unused_keys.is_empty() { - return Err(format!("Unused keys in header: {}", unused_keys.join(", ")).into()); - } - let missing_keys = header - .missing() - .iter() - .map(|s| s.as_str()) - .collect::>(); - if !missing_keys.is_empty() { - return Err(format!( - "Required values were missing from the header: {}", - missing_keys.join(", ") - ) - .into()); - } + src_env.check_unused_header_keys()?; self.output_file( &grm, &stable, - &derived_mod_name, + build_env.derived_mod_name(), outp, &format!("/* CACHE INFORMATION {} */\n", cache), - &yacc_diag, + src_env.yacc_diag(), + &build_env, )?; let conflicts = if stable.conflicts().is_some() { Some((sgraph, stable)) @@ -1084,6 +1019,7 @@ where outp_rs: P, cache: &str, diag: &SpannedDiagnosticFormatter, + build_env: &ParserBuildEnv<'_, LexerTypesT>, ) -> Result<(), Box> { let visibility = self.visibility.clone(); let user_actions = if let Some( @@ -1096,7 +1032,7 @@ where }; let rule_consts = self.gen_rule_consts(grm)?; let token_epp = self.gen_token_epp(grm)?; - let parse_function = self.gen_parse_function(grm, stable)?; + let parse_function = self.gen_parse_function(grm, stable, build_env)?; let action_wrappers = match self.yacckind.unwrap() { YaccKind::Original(YaccOriginalActionKind::UserAction) | YaccKind::Grmtools => { Some(self.gen_wrappers(grm)?) @@ -1234,6 +1170,7 @@ where &self, grm: &YaccGrammar, stable: &StateTable, + build_env: &ParserBuildEnv<'_, LexerTypesT>, ) -> Result> { let storaget = str::parse::(type_name::())?; let lexertypest = str::parse::(type_name::())?; @@ -1347,9 +1284,7 @@ where _ => unreachable!(), }; - let serialisation_format = self - .serialisation_format - .expect("Should already have a default value"); + let serialisation_format = build_env.serialisation_format(); // Note that the configuration types use associated consts, and thus these configurations represent distinct types. let (grm_data, stable_data): (Vec, Vec) = match serialisation_format { SerialisationFormat::FixedSizeInteger => { @@ -1904,7 +1839,7 @@ where /// /// It is plausible that we should a step 4, but currently do not: /// 4. Replace all `\n{indent}\n` with `\n\n` -fn indent(indent: &str, s: &str) -> String { +pub(crate) fn indent(indent: &str, s: &str) -> String { format!("{indent}{}\n", s.trim_end_matches('\n')).replace('\n', &format!("\n{}", indent)) } diff --git a/lrpar/src/lib/mod.rs b/lrpar/src/lib/mod.rs index 00533978e..a9ed046f3 100644 --- a/lrpar/src/lib/mod.rs +++ b/lrpar/src/lib/mod.rs @@ -205,6 +205,8 @@ pub mod parser; #[cfg(test)] pub mod test_utils; +mod codegen; + pub use crate::{ ctbuilder::{ CTParser, CTParserBuilder, ParserData, RustEdition, SerialisationFormat, Visibility, From dd71ab075d33f005907dd48b0dd29fcbb6b397ee Mon Sep 17 00:00:00 2001 From: matt rice Date: Thu, 6 Aug 2026 00:37:46 -0700 Subject: [PATCH 02/14] Build the grm, state graph, and state table as part of the code_gen struct. --- lrpar/src/lib/codegen.rs | 109 +++++++++++++++++++++++++++++++++++-- lrpar/src/lib/ctbuilder.rs | 79 +++++++-------------------- 2 files changed, 123 insertions(+), 65 deletions(-) diff --git a/lrpar/src/lib/codegen.rs b/lrpar/src/lib/codegen.rs index 265d3a0e5..e5da206d7 100644 --- a/lrpar/src/lib/codegen.rs +++ b/lrpar/src/lib/codegen.rs @@ -1,11 +1,11 @@ #![deny(unfulfilled_lint_expectations)] #![expect(dead_code)] -use std::{error::Error, fmt, marker::PhantomData, path::Path}; +use std::{error::Error, fmt, hash::Hash, marker::PhantomData, path::Path}; use crate::{ LexerTypes, RecoveryKind, RustEdition, SerialisationFormat, Visibility, - ctbuilder::{ERROR, indent}, + ctbuilder::{CTConflictsError, ERROR, FixIntConfig, VarIntConfig, indent}, diagnostics::{DiagnosticFormatter, SpannedDiagnosticFormatter}, }; @@ -14,7 +14,9 @@ use cfgrammar::{ header::{GrmtoolsSectionParser, Header, HeaderValue}, yacc::{YaccGrammar, YaccKind, ast::ASTWithValidityInfo}, }; -use lrtable::{StateGraph, StateTable}; + +use lrtable::{Minimiser, StateGraph, StateTable, from_yacc}; +use wincode::SchemaWrite; pub(crate) struct ParserSrcEnv<'a> { src: &'a str, @@ -216,9 +218,7 @@ impl<'a> ParserSrcEnv<'a> { } } - fn extract_serialisation_format( - &mut self, - ) -> Result> { + fn extract_serialisation_format(&mut self) -> Result> { self.header.mark_used(&"serialisation_format".to_string()); if let Some(ec_val) = self .header @@ -297,6 +297,103 @@ where pub(crate) fn recoverer(&self) -> RecoveryKind { self.recoverer } + + pub(crate) fn code_generator( + &self, + src_env: &ParserSrcEnv, + timestamp: &str, + ) -> Result, Box> { + let grm = match YaccGrammar::::new_from_ast_with_validity_info( + &self.ast_validation, + ) { + Ok(grm) => grm, + Err(errs) => { + let mut out = String::new(); + out.push_str(&format!( + "\n{ERROR}{}\n", + src_env.yacc_diag().file_location_msg("", None) + )); + for e in errs { + out.push_str(&indent( + " ", + &src_env.yacc_diag().format_error(e).to_string(), + )); + out.push('\n'); + } + return Err(ErrorString(out).into()); + } + }; + + let (sgraph, stable) = from_yacc(&grm, Minimiser::Pager)?; + if self.cache_args.error_on_conflicts + && let Some(c) = stable.conflicts() + { + match (grm.expect(), grm.expectrr()) { + (Some(i), Some(j)) if i == c.sr_len() && j == c.rr_len() => (), + (Some(i), None) if i == c.sr_len() && 0 == c.rr_len() => (), + (None, Some(j)) if 0 == c.sr_len() && j == c.rr_len() => (), + (None, None) if 0 == c.rr_len() && 0 == c.sr_len() => (), + _ => { + let conflicts_diagnostic = src_env.yacc_diag().format_conflicts::( + &grm, + self.ast_validation.ast(), + c, + &sgraph, + &stable, + ); + return Err(Box::new(CTConflictsError { + conflicts_diagnostic, + phantom: PhantomData, + #[cfg(test)] + stable, + })); + } + } + } + + Ok(ParserCodegen { + grm, + stable, + sgraph, + timestamp: timestamp.to_string(), + }) + } +} + +impl ParserCodegen +where + LexerTypesT: LexerTypes, + usize: num_traits::AsPrimitive, + LexerTypesT::StorageT: 'static + + fmt::Debug + + Hash + + num_traits::PrimInt + + SchemaWrite + + SchemaWrite + + num_traits::Unsigned, + LexerTypesT: LexerTypes, +{ + pub(crate) fn grm(&self) -> &YaccGrammar { + &self.grm + } + + pub(crate) fn stable(&self) -> &StateTable { + &self.stable + } + + pub(crate) fn sgraph(&self) -> &StateGraph { + &self.sgraph + } + + pub(crate) fn take_parser( + self, + ) -> ( + YaccGrammar, + StateGraph, + StateTable, + ) { + (self.grm, self.sgraph, self.stable) + } } /// A string which uses `Display` for it's `Debug` impl. diff --git a/lrpar/src/lib/ctbuilder.rs b/lrpar/src/lib/ctbuilder.rs index 240aa44ca..f67e372d9 100644 --- a/lrpar/src/lib/ctbuilder.rs +++ b/lrpar/src/lib/ctbuilder.rs @@ -30,7 +30,7 @@ use cfgrammar::{ yacc::{YaccGrammar, YaccKind, YaccOriginalActionKind, ast::ASTWithValidityInfo}, }; use filetime::FileTime; -use lrtable::{Minimiser, StateGraph, StateTable, from_yacc, statetable::Conflicts}; +use lrtable::{StateGraph, StateTable, statetable::Conflicts}; use num_traits::{AsPrimitive, PrimInt, Unsigned}; use proc_macro2::{Literal, TokenStream}; use quote::{ToTokens, TokenStreamExt, format_ident, quote}; @@ -51,12 +51,12 @@ pub(crate) const ERROR: &str = "[Error]"; static GENERATED_PATHS: LazyLock>> = LazyLock::new(|| Mutex::new(HashSet::new())); -struct CTConflictsError { - conflicts_diagnostic: String, +pub(crate) struct CTConflictsError { + pub(crate) conflicts_diagnostic: String, #[cfg(test)] #[cfg_attr(test, allow(dead_code))] - stable: StateTable, - phantom: PhantomData, + pub(crate) stable: StateTable, + pub(crate) phantom: PhantomData, } /// The quote impl of `ToTokens` for `Option` prints an empty string for `None` @@ -367,9 +367,9 @@ where } /// Defaults to `wincode::int_encoding::VarInt`. -type FixIntConfig = wincode::config::Configuration; +pub(crate) type FixIntConfig = wincode::config::Configuration; /// The default config with the last parameter set to `VarInt` -type VarIntConfig = wincode::config::Configuration< +pub(crate) type VarIntConfig = wincode::config::Configuration< true, 4194304, wincode::len::BincodeLen, @@ -768,38 +768,24 @@ where } } } - let grm = match YaccGrammar::::new_from_ast_with_validity_info( - build_env.ast_validation(), - ) { - Ok(grm) => grm, - Err(errs) => { - let mut out = String::new(); - out.push_str(&format!( - "\n{ERROR}{}\n", - src_env.yacc_diag().file_location_msg("", None) - )); - for e in errs { - out.push_str(&indent( - " ", - &src_env.yacc_diag().format_error(e).to_string(), - )); - out.push('\n'); - } - return Err(ErrorString(out).into()); - } - }; + #[cfg(test)] if let Some(cb) = &self.inspect_callback { cb(build_env.recoverer())?; } + let timestamp = env!("VERGEN_BUILD_TIMESTAMP"); + let code_gen = build_env.code_generator(&src_env, timestamp)?; + let grm = code_gen.grm(); + let stable = code_gen.stable(); + let rule_ids = grm .tokens_map() .iter() .map(|(&n, &i)| (n.to_owned(), i.as_storaget())) .collect::>(); - let cache = self.rebuild_cache(build_env.derived_mod_name(), &grm); + let cache = self.rebuild_cache(build_env.derived_mod_name(), grm); // We don't need to go through the full rigmarole of generating an output file if all of // the following are true: the output file exists; it is newer than the input file; and the @@ -815,6 +801,8 @@ where && let Ok(outc) = read_to_string(outp) { if outc.contains(&cache.to_string()) { + let (grm, _, _) = code_gen.take_parser(); + return Ok(CTParser { regenerated: false, rule_ids, @@ -842,36 +830,8 @@ where // confusing than the alternatives). fs::remove_file(outp).ok(); - let (sgraph, stable) = from_yacc(&grm, Minimiser::Pager)?; - if self.error_on_conflicts - && let Some(c) = stable.conflicts() - { - match (grm.expect(), grm.expectrr()) { - (Some(i), Some(j)) if i == c.sr_len() && j == c.rr_len() => (), - (Some(i), None) if i == c.sr_len() && 0 == c.rr_len() => (), - (None, Some(j)) if 0 == c.sr_len() && j == c.rr_len() => (), - (None, None) if 0 == c.rr_len() && 0 == c.sr_len() => (), - _ => { - let conflicts_diagnostic = src_env.yacc_diag().format_conflicts::( - &grm, - build_env.ast_validation().ast(), - c, - &sgraph, - &stable, - ); - return Err(Box::new(CTConflictsError { - conflicts_diagnostic, - phantom: PhantomData, - #[cfg(test)] - stable, - })); - } - } - } - if let Some(ref mut inspector_rt) = self.inspect_rt { - let rt: RTParserBuilder<'_, StorageT, LexerTypesT> = - RTParserBuilder::new(&grm, &stable); + let rt: RTParserBuilder<'_, StorageT, LexerTypesT> = RTParserBuilder::new(grm, stable); let rt = if let Some(rk) = self.recoverer { rt.recoverer(rk) } else { @@ -883,14 +843,15 @@ where src_env.check_unused_header_keys()?; self.output_file( - &grm, - &stable, + grm, + stable, build_env.derived_mod_name(), outp, &format!("/* CACHE INFORMATION {} */\n", cache), src_env.yacc_diag(), &build_env, )?; + let (grm, sgraph, stable) = code_gen.take_parser(); let conflicts = if stable.conflicts().is_some() { Some((sgraph, stable)) } else { From d51e3b78387bc9d191f874dd9da616f95497256a Mon Sep 17 00:00:00 2001 From: matt rice Date: Thu, 6 Aug 2026 18:24:06 -0700 Subject: [PATCH 03/14] Prepare `rebuild_cache` to be moved --- lrpar/src/lib/codegen.rs | 28 ++++++++++++++++++ lrpar/src/lib/ctbuilder.rs | 58 +++++++++++++------------------------- 2 files changed, 48 insertions(+), 38 deletions(-) diff --git a/lrpar/src/lib/codegen.rs b/lrpar/src/lib/codegen.rs index e5da206d7..0c8d2e410 100644 --- a/lrpar/src/lib/codegen.rs +++ b/lrpar/src/lib/codegen.rs @@ -124,6 +124,10 @@ impl<'a> ParserSrcEnv<'a> { } } + pub(crate) fn path(&self) -> &Path { + self.path + } + pub(crate) fn yacc_diag(&self) -> &SpannedDiagnosticFormatter<'a> { &self.diagnostics } @@ -294,10 +298,34 @@ where &self.mod_name } + pub(crate) fn specified_mod_name(&self) -> Option<&str> { + self.cache_args.mod_name.as_deref() + } + pub(crate) fn recoverer(&self) -> RecoveryKind { self.recoverer } + pub(crate) fn rust_edition(&self) -> RustEdition { + self.cache_args.rust_edition + } + + pub(crate) fn visibility(&self) -> &Visibility { + &self.cache_args.visibility + } + + pub(crate) fn show_warnings(&self) -> bool { + self.cache_args.show_warnings + } + + pub(crate) fn warnings_are_errors(&self) -> bool { + self.cache_args.warnings_are_errors + } + + pub(crate) fn error_on_conflicts(&self) -> bool { + self.cache_args.error_on_conflicts + } + pub(crate) fn code_generator( &self, src_env: &ParserSrcEnv, diff --git a/lrpar/src/lib/ctbuilder.rs b/lrpar/src/lib/ctbuilder.rs index f67e372d9..bc5b8f74d 100644 --- a/lrpar/src/lib/ctbuilder.rs +++ b/lrpar/src/lib/ctbuilder.rs @@ -16,7 +16,7 @@ use std::{ use crate::{ LexerTypes, RTParserBuilder, RecoveryKind, - codegen::{ParserBuildEnv, ParserBuildEnvArgs, ParserSrcEnv}, + codegen::{ParserBuildEnv, ParserBuildEnvArgs, ParserCodegen, ParserSrcEnv}, diagnostics::{DiagnosticFormatter, SpannedDiagnosticFormatter}, }; @@ -785,7 +785,7 @@ where .map(|(&n, &i)| (n.to_owned(), i.as_storaget())) .collect::>(); - let cache = self.rebuild_cache(build_env.derived_mod_name(), grm); + let cache = self.rebuild_cache(&code_gen, &src_env, &build_env); // We don't need to go through the full rigmarole of generating an output file if all of // the following are true: the output file exists; it is newer than the input file; and the @@ -1061,42 +1061,19 @@ where /// Generate the cache, which determines if anything's changed enough that we need to /// regenerate outputs and force rustc to recompile. - fn rebuild_cache(&self, derived_mod_name: &'_ str, grm: &YaccGrammar) -> TokenStream { - // We don't need to be particularly clever here: we just need to record the various things - // that could change between builds. - // - // Record the time that this version of lrpar was built. If the source code changes and - // rustc forces a recompile, this will change this value, causing anything which depends on - // this build of lrpar to be recompiled too. - let Self { - // All variables except for `output_path`, `inspect_callback` and `phantom` should - // be written into the cache. - grammar_path, - // I struggle to imagine the correct thing for `grammar_src`. - grammar_src: _, - // I struggle to imagine the correct thing for `from_ast`. - from_ast: _, - mod_name, - recoverer, - yacckind, - output_path: _, - error_on_conflicts, - warnings_are_errors, - show_warnings, - visibility, - rust_edition, - serialisation_format, - inspect_rt: _, - #[cfg(test)] - inspect_callback: _, - phantom: _, - } = self; + fn rebuild_cache( + &self, + code_gen: &ParserCodegen, + src_env: &ParserSrcEnv, + build_env: &ParserBuildEnv, + ) -> TokenStream { + let grm = code_gen.grm(); let build_time = env!("VERGEN_BUILD_TIMESTAMP"); - let grammar_path = grammar_path.as_ref().unwrap().to_string_lossy(); - let mod_name = QuoteOption(mod_name.as_deref()); - let visibility = visibility.to_variant_tokens(); - let rust_edition = rust_edition.to_variant_tokens(); - let yacckind = yacckind.expect("is_some() by this point"); + let grammar_path = src_env.path().to_string_lossy(); + let mod_name = QuoteOption(build_env.specified_mod_name()); + let visibility = build_env.visibility().to_variant_tokens(); + let rust_edition = build_env.rust_edition().to_variant_tokens(); + let yacckind = build_env.ast_validation().yacc_kind(); let rule_map = grm .iter_tidxs() .map(|tidx| { @@ -1106,6 +1083,12 @@ where )) }) .collect::>(); + let derived_mod_name = build_env.derived_mod_name(); + let serialisation_format = build_env.serialisation_format(); + let recoverer = build_env.recoverer(); + let error_on_conflicts = build_env.error_on_conflicts(); + let show_warnings = build_env.show_warnings(); + let warnings_are_errors = build_env.warnings_are_errors(); let cache_info = quote! { BUILD_TIME = #build_time DERIVED_MOD_NAME = #derived_mod_name @@ -1120,7 +1103,6 @@ where RUST_EDITION = #rust_edition RULE_IDS_MAP = [#(#rule_map,)*] VISIBILITY = #visibility - }; let cache_info_str = cache_info.to_string(); quote!(#cache_info_str) From b8c07556081a2ee9ef54eecb8e6c080d241488a9 Mon Sep 17 00:00:00 2001 From: matt rice Date: Thu, 6 Aug 2026 18:40:57 -0700 Subject: [PATCH 04/14] Move `rebuild_cache` to the codegen module --- lrpar/src/lib/codegen.rs | 138 +++++++++++++++++++++++++++++++++++++ lrpar/src/lib/ctbuilder.rs | 134 +---------------------------------- 2 files changed, 141 insertions(+), 131 deletions(-) diff --git a/lrpar/src/lib/codegen.rs b/lrpar/src/lib/codegen.rs index 0c8d2e410..5923f6892 100644 --- a/lrpar/src/lib/codegen.rs +++ b/lrpar/src/lib/codegen.rs @@ -16,6 +16,8 @@ use cfgrammar::{ }; use lrtable::{Minimiser, StateGraph, StateTable, from_yacc}; +use proc_macro2::TokenStream; +use quote::{ToTokens, TokenStreamExt, quote}; use wincode::SchemaWrite; pub(crate) struct ParserSrcEnv<'a> { @@ -422,6 +424,62 @@ where ) { (self.grm, self.sgraph, self.stable) } + + /// Generate the cache, which determines if anything's changed enough that we need to + /// regenerate outputs and force rustc to recompile. + fn gen_cache( + &self, + src_env: &ParserSrcEnv, + build_env: &ParserBuildEnv, + ) -> TokenStream { + let grm = self.grm(); + let build_time = env!("VERGEN_BUILD_TIMESTAMP"); + let grammar_path = src_env.path().to_string_lossy(); + let mod_name = QuoteOption(build_env.specified_mod_name()); + let visibility = build_env.visibility().to_variant_tokens(); + let rust_edition = build_env.rust_edition().to_variant_tokens(); + let yacckind = build_env.ast_validation().yacc_kind(); + let rule_map = grm + .iter_tidxs() + .map(|tidx| { + QuoteTuple(( + usize::from(tidx), + grm.token_name(tidx).unwrap_or(""), + )) + }) + .collect::>(); + let derived_mod_name = build_env.derived_mod_name(); + let serialisation_format = build_env.serialisation_format(); + let recoverer = build_env.recoverer(); + let error_on_conflicts = build_env.error_on_conflicts(); + let show_warnings = build_env.show_warnings(); + let warnings_are_errors = build_env.warnings_are_errors(); + let cache_info = quote! { + BUILD_TIME = #build_time + DERIVED_MOD_NAME = #derived_mod_name + ENCODING_CONFIG = #serialisation_format + GRAMMAR_PATH = #grammar_path + MOD_NAME = #mod_name + RECOVERER = #recoverer + YACC_KIND = #yacckind + ERROR_ON_CONFLICTS = #error_on_conflicts + SHOW_WARNINGS = #show_warnings + WARNINGS_ARE_ERRORS = #warnings_are_errors + RUST_EDITION = #rust_edition + RULE_IDS_MAP = [#(#rule_map,)*] + VISIBILITY = #visibility + }; + let cache_info_str = cache_info.to_string(); + quote!(#cache_info_str) + } + + pub(crate) fn cache_str( + &self, + src_env: &ParserSrcEnv, + build_env: &ParserBuildEnv, + ) -> String { + self.gen_cache(src_env, build_env).to_string() + } } /// A string which uses `Display` for it's `Debug` impl. @@ -439,3 +497,83 @@ impl fmt::Debug for ErrorString { } } impl Error for ErrorString {} + +/// The quote impl of `ToTokens` for `Option` prints an empty string for `None` +/// and the inner value for `Some(inner_value)`. +/// +/// This wrapper instead emits both `Some` and `None` variants. +/// See: [quote #20](https://github.com/dtolnay/quote/issues/20) +// Temporarily pub(crate) +pub(crate) struct QuoteOption(pub(crate) Option); + +impl ToTokens for QuoteOption { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.append_all(match self.0 { + Some(ref t) => quote! { ::std::option::Option::Some(#t) }, + None => quote! { ::std::option::Option::None }, + }); + } +} + +/// This wrapper adds a missing impl of `ToTokens` for tuples. +/// For a tuple `(a, b)` emits `(a.to_tokens(), b.to_tokens())` +struct QuoteTuple(T); + +impl ToTokens for QuoteTuple<(A, B)> { + fn to_tokens(&self, tokens: &mut TokenStream) { + let (a, b) = &self.0; + tokens.append_all(quote!((#a, #b))); + } +} + +/// The wrapped `&str` value will be emitted with a call to `to_string()` +struct QuoteToString<'a>(&'a str); + +impl ToTokens for QuoteToString<'_> { + fn to_tokens(&self, tokens: &mut TokenStream) { + let x = &self.0; + tokens.append_all(quote! { #x.to_string() }); + } +} + +impl RustEdition { + fn to_variant_tokens(self) -> TokenStream { + match self { + RustEdition::Rust2015 => quote!(::lrpar::RustEdition::Rust2015), + RustEdition::Rust2018 => quote!(::lrpar::RustEdition::Rust2018), + RustEdition::Rust2021 => quote!(::lrpar::RustEdition::Rust2021), + } + } +} + +impl ToTokens for Visibility { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.extend(match self { + Visibility::Private => quote!(), + Visibility::Public => quote! {pub}, + Visibility::PublicSuper => quote! {pub(super)}, + Visibility::PublicSelf => quote! {pub(self)}, + Visibility::PublicCrate => quote! {pub(crate)}, + Visibility::PublicIn(data) => { + let other = str::parse::(data).unwrap(); + quote! {pub(in #other)} + } + }) + } +} + +impl Visibility { + fn to_variant_tokens(&self) -> TokenStream { + match self { + Visibility::Private => quote!(::lrpar::Visibility::Private), + Visibility::Public => quote!(::lrpar::Visibility::Public), + Visibility::PublicSuper => quote!(::lrpar::Visibility::PublicSuper), + Visibility::PublicSelf => quote!(::lrpar::Visibility::PublicSelf), + Visibility::PublicCrate => quote!(::lrpar::Visibility::PublicCrate), + Visibility::PublicIn(data) => { + let data = QuoteToString(data); + quote!(::lrpar::Visibility::PublicIn(#data)) + } + } + } +} diff --git a/lrpar/src/lib/ctbuilder.rs b/lrpar/src/lib/ctbuilder.rs index bc5b8f74d..a8a4a2795 100644 --- a/lrpar/src/lib/ctbuilder.rs +++ b/lrpar/src/lib/ctbuilder.rs @@ -16,7 +16,7 @@ use std::{ use crate::{ LexerTypes, RTParserBuilder, RecoveryKind, - codegen::{ParserBuildEnv, ParserBuildEnvArgs, ParserCodegen, ParserSrcEnv}, + codegen::{ParserBuildEnv, ParserBuildEnvArgs, ParserSrcEnv, QuoteOption}, diagnostics::{DiagnosticFormatter, SpannedDiagnosticFormatter}, }; @@ -59,22 +59,6 @@ pub(crate) struct CTConflictsError { pub(crate) phantom: PhantomData, } -/// The quote impl of `ToTokens` for `Option` prints an empty string for `None` -/// and the inner value for `Some(inner_value)`. -/// -/// This wrapper instead emits both `Some` and `None` variants. -/// See: [quote #20](https://github.com/dtolnay/quote/issues/20) -struct QuoteOption(Option); - -impl ToTokens for QuoteOption { - fn to_tokens(&self, tokens: &mut TokenStream) { - tokens.append_all(match self.0 { - Some(ref t) => quote! { ::std::option::Option::Some(#t) }, - None => quote! { ::std::option::Option::None }, - }); - } -} - /// The quote impl of `ToTokens` for `usize` prints literal values /// including a type suffix for example `0usize`. /// @@ -87,27 +71,6 @@ impl ToTokens for UnsuffixedUsize { } } -/// This wrapper adds a missing impl of `ToTokens` for tuples. -/// For a tuple `(a, b)` emits `(a.to_tokens(), b.to_tokens())` -struct QuoteTuple(T); - -impl ToTokens for QuoteTuple<(A, B)> { - fn to_tokens(&self, tokens: &mut TokenStream) { - let (a, b) = &self.0; - tokens.append_all(quote!((#a, #b))); - } -} - -/// The wrapped `&str` value will be emitted with a call to `to_string()` -struct QuoteToString<'a>(&'a str); - -impl ToTokens for QuoteToString<'_> { - fn to_tokens(&self, tokens: &mut TokenStream) { - let x = &self.0; - tokens.append_all(quote! { #x.to_string() }); - } -} - impl fmt::Display for CTConflictsError where StorageT: 'static + Debug + Hash + PrimInt + Unsigned, @@ -180,48 +143,6 @@ pub enum RustEdition { Rust2021, } -impl RustEdition { - fn to_variant_tokens(self) -> TokenStream { - match self { - RustEdition::Rust2015 => quote!(::lrpar::RustEdition::Rust2015), - RustEdition::Rust2018 => quote!(::lrpar::RustEdition::Rust2018), - RustEdition::Rust2021 => quote!(::lrpar::RustEdition::Rust2021), - } - } -} - -impl ToTokens for Visibility { - fn to_tokens(&self, tokens: &mut TokenStream) { - tokens.extend(match self { - Visibility::Private => quote!(), - Visibility::Public => quote! {pub}, - Visibility::PublicSuper => quote! {pub(super)}, - Visibility::PublicSelf => quote! {pub(self)}, - Visibility::PublicCrate => quote! {pub(crate)}, - Visibility::PublicIn(data) => { - let other = str::parse::(data).unwrap(); - quote! {pub(in #other)} - } - }) - } -} - -impl Visibility { - fn to_variant_tokens(&self) -> TokenStream { - match self { - Visibility::Private => quote!(::lrpar::Visibility::Private), - Visibility::Public => quote!(::lrpar::Visibility::Public), - Visibility::PublicSuper => quote!(::lrpar::Visibility::PublicSuper), - Visibility::PublicSelf => quote!(::lrpar::Visibility::PublicSelf), - Visibility::PublicCrate => quote!(::lrpar::Visibility::PublicCrate), - Visibility::PublicIn(data) => { - let data = QuoteToString(data); - quote!(::lrpar::Visibility::PublicIn(#data)) - } - } - } -} - /// Sets the underlying encoding algorithm for serialising the `ParserData` into the generated source files. /// /// This correlates to a specific `Configuration` of [wincode::config](https://docs.rs/wincode/latest/wincode/config/index.html). @@ -785,7 +706,7 @@ where .map(|(&n, &i)| (n.to_owned(), i.as_storaget())) .collect::>(); - let cache = self.rebuild_cache(&code_gen, &src_env, &build_env); + let cache = code_gen.cache_str(&src_env, &build_env); // We don't need to go through the full rigmarole of generating an output file if all of // the following are true: the output file exists; it is newer than the input file; and the @@ -800,7 +721,7 @@ where > FileTime::from_last_modification_time(inmd) && let Ok(outc) = read_to_string(outp) { - if outc.contains(&cache.to_string()) { + if outc.contains(&cache) { let (grm, _, _) = code_gen.take_parser(); return Ok(CTParser { @@ -1059,55 +980,6 @@ where Ok(()) } - /// Generate the cache, which determines if anything's changed enough that we need to - /// regenerate outputs and force rustc to recompile. - fn rebuild_cache( - &self, - code_gen: &ParserCodegen, - src_env: &ParserSrcEnv, - build_env: &ParserBuildEnv, - ) -> TokenStream { - let grm = code_gen.grm(); - let build_time = env!("VERGEN_BUILD_TIMESTAMP"); - let grammar_path = src_env.path().to_string_lossy(); - let mod_name = QuoteOption(build_env.specified_mod_name()); - let visibility = build_env.visibility().to_variant_tokens(); - let rust_edition = build_env.rust_edition().to_variant_tokens(); - let yacckind = build_env.ast_validation().yacc_kind(); - let rule_map = grm - .iter_tidxs() - .map(|tidx| { - QuoteTuple(( - usize::from(tidx), - grm.token_name(tidx).unwrap_or(""), - )) - }) - .collect::>(); - let derived_mod_name = build_env.derived_mod_name(); - let serialisation_format = build_env.serialisation_format(); - let recoverer = build_env.recoverer(); - let error_on_conflicts = build_env.error_on_conflicts(); - let show_warnings = build_env.show_warnings(); - let warnings_are_errors = build_env.warnings_are_errors(); - let cache_info = quote! { - BUILD_TIME = #build_time - DERIVED_MOD_NAME = #derived_mod_name - ENCODING_CONFIG = #serialisation_format - GRAMMAR_PATH = #grammar_path - MOD_NAME = #mod_name - RECOVERER = #recoverer - YACC_KIND = #yacckind - ERROR_ON_CONFLICTS = #error_on_conflicts - SHOW_WARNINGS = #show_warnings - WARNINGS_ARE_ERRORS = #warnings_are_errors - RUST_EDITION = #rust_edition - RULE_IDS_MAP = [#(#rule_map,)*] - VISIBILITY = #visibility - }; - let cache_info_str = cache_info.to_string(); - quote!(#cache_info_str) - } - /// Generate the main parse() function for the output file. fn gen_parse_function( &self, From 653a2857ec4604e47240629f300b16ae88188e2a Mon Sep 17 00:00:00 2001 From: matt rice Date: Thu, 6 Aug 2026 19:22:14 -0700 Subject: [PATCH 05/14] Prepare `output_file` and friends to be moved --- lrpar/src/lib/codegen.rs | 4 ++ lrpar/src/lib/ctbuilder.rs | 123 ++++++++++++++++++++----------------- 2 files changed, 70 insertions(+), 57 deletions(-) diff --git a/lrpar/src/lib/codegen.rs b/lrpar/src/lib/codegen.rs index 5923f6892..7d340c3d1 100644 --- a/lrpar/src/lib/codegen.rs +++ b/lrpar/src/lib/codegen.rs @@ -328,6 +328,10 @@ where self.cache_args.error_on_conflicts } + pub(crate) fn yacc_kind(&self) -> YaccKind { + self.ast_validation().yacc_kind() + } + pub(crate) fn code_generator( &self, src_env: &ParserSrcEnv, diff --git a/lrpar/src/lib/ctbuilder.rs b/lrpar/src/lib/ctbuilder.rs index a8a4a2795..0bf9d1fec 100644 --- a/lrpar/src/lib/ctbuilder.rs +++ b/lrpar/src/lib/ctbuilder.rs @@ -16,8 +16,8 @@ use std::{ use crate::{ LexerTypes, RTParserBuilder, RecoveryKind, - codegen::{ParserBuildEnv, ParserBuildEnvArgs, ParserSrcEnv, QuoteOption}, - diagnostics::{DiagnosticFormatter, SpannedDiagnosticFormatter}, + codegen::{ParserBuildEnv, ParserBuildEnvArgs, ParserCodegen, ParserSrcEnv, QuoteOption}, + diagnostics::DiagnosticFormatter, }; #[cfg(feature = "_unstable_api")] @@ -764,12 +764,10 @@ where src_env.check_unused_header_keys()?; self.output_file( - grm, - stable, - build_env.derived_mod_name(), + &code_gen, outp, &format!("/* CACHE INFORMATION {} */\n", cache), - src_env.yacc_diag(), + &src_env, &build_env, )?; let (grm, sgraph, stable) = code_gen.take_parser(); @@ -895,48 +893,46 @@ where fn output_file>( &self, - grm: &YaccGrammar, - stable: &StateTable, - mod_name: &str, + code_gen: &ParserCodegen, outp_rs: P, cache: &str, - diag: &SpannedDiagnosticFormatter, + src_env: &ParserSrcEnv, build_env: &ParserBuildEnv<'_, LexerTypesT>, ) -> Result<(), Box> { - let visibility = self.visibility.clone(); - let user_actions = if let Some( - YaccKind::Original(YaccOriginalActionKind::UserAction) | YaccKind::Grmtools, - ) = self.yacckind + let mod_name = build_env.derived_mod_name(); + let visibility = build_env.visibility(); + let user_actions = if let YaccKind::Original(YaccOriginalActionKind::UserAction) + | YaccKind::Grmtools = build_env.yacc_kind() { - Some(self.gen_user_actions(grm, diag)?) + Some(self.gen_user_actions(code_gen, src_env)?) } else { None }; - let rule_consts = self.gen_rule_consts(grm)?; - let token_epp = self.gen_token_epp(grm)?; - let parse_function = self.gen_parse_function(grm, stable, build_env)?; - let action_wrappers = match self.yacckind.unwrap() { + + let rule_consts = self.gen_rule_consts(code_gen)?; + let token_epp = self.gen_token_epp(code_gen)?; + let parse_function = self.gen_parse_function(code_gen, build_env)?; + let action_wrappers = match build_env.yacc_kind() { YaccKind::Original(YaccOriginalActionKind::UserAction) | YaccKind::Grmtools => { - Some(self.gen_wrappers(grm)?) + Some(self.gen_wrappers(code_gen, build_env)?) } YaccKind::Original(YaccOriginalActionKind::NoAction) | YaccKind::Original(YaccOriginalActionKind::GenericParseTree) => None, _ => unreachable!(), }; - let additional_decls = - if let Some(YaccKind::Original(YaccOriginalActionKind::GenericParseTree)) = - self.yacckind - { - // `lrpar::Node`` is deprecated within the lrpar crate, but not from within this module, - // Once it is removed from `lrpar`, we should move the declaration here entirely. - Some(quote! { - #[allow(unused_imports)] - pub use ::lrpar::parser::_deprecated_moved_::Node; - }) - } else { - None - }; + let additional_decls = if let YaccKind::Original(YaccOriginalActionKind::GenericParseTree) = + build_env.yacc_kind() + { + // `lrpar::Node`` is deprecated within the lrpar crate, but not from within this module, + // Once it is removed from `lrpar`, we should move the declaration here entirely. + Some(quote! { + #[allow(unused_imports)] + pub use ::lrpar::parser::_deprecated_moved_::Node; + }) + } else { + None + }; let mod_name = match syn::parse_str::(mod_name) { @@ -983,14 +979,15 @@ where /// Generate the main parse() function for the output file. fn gen_parse_function( &self, - grm: &YaccGrammar, - stable: &StateTable, - build_env: &ParserBuildEnv<'_, LexerTypesT>, + code_gen: &ParserCodegen, + build_env: &ParserBuildEnv, ) -> Result> { - let storaget = str::parse::(type_name::())?; + let stable = code_gen.stable(); + let grm = code_gen.grm(); + let storaget = str::parse::(type_name::())?; let lexertypest = str::parse::(type_name::())?; - let recoverer = self.recoverer; - let run_parser = match self.yacckind.unwrap() { + let recoverer = build_env.recoverer(); + let run_parser = match build_env.ast_validation().yacc_kind() { YaccKind::Original(YaccOriginalActionKind::GenericParseTree) => { quote! { ::lrpar::RTParserBuilder::new(grm, stable) @@ -1027,12 +1024,12 @@ where let pidx = usize::from(pidx); format_ident!("{}wrapper_{}", ACTION_PREFIX, pidx) }); - let edition_lifetime = if self.rust_edition != RustEdition::Rust2015 { + let edition_lifetime = if build_env.rust_edition() != RustEdition::Rust2015 { quote!('_,) } else { quote!() }; - let ridx = usize::from(self.user_start_ridx(grm)); + let ridx = usize::from(self.user_start_ridx(code_gen)); let action_ident = format_ident!("{}{}", ACTIONS_KIND_PREFIX, ridx); quote! { @@ -1057,7 +1054,7 @@ where kind => panic!("YaccKind {:?} not supported", kind), }; - let parsed_parse_generics: Generics = match self.yacckind.unwrap() { + let parsed_parse_generics: Generics = match build_env.ast_validation().yacc_kind() { YaccKind::Original(YaccOriginalActionKind::UserAction) | YaccKind::Grmtools => { make_generics(grm.parse_generics().as_deref())? } @@ -1066,7 +1063,7 @@ where let (generics, _, where_clause) = parsed_parse_generics.split_for_impl(); // `parse()` may or may not have an argument for `%parseparam`. - let parse_fn_parse_param = match self.yacckind.unwrap() { + let parse_fn_parse_param = match build_env.ast_validation().yacc_kind() { YaccKind::Original(YaccOriginalActionKind::UserAction) | YaccKind::Grmtools => { if let Some((name, tyname)) = grm.parse_param() { let name = str::parse::(name)?; @@ -1078,10 +1075,10 @@ where } _ => None, }; - let parse_fn_return_ty = match self.yacckind.unwrap() { + let parse_fn_return_ty = match build_env.ast_validation().yacc_kind() { YaccKind::Original(YaccOriginalActionKind::UserAction) | YaccKind::Grmtools => { let actiont = grm - .actiontype(self.user_start_ridx(grm)) + .actiontype(self.user_start_ridx(code_gen)) .as_ref() .map(|at| str::parse::(at)) .transpose()?; @@ -1160,13 +1157,14 @@ where fn gen_rule_consts( &self, - grm: &YaccGrammar, + code_gen: &ParserCodegen, ) -> Result { + let grm = code_gen.grm(); let mut toks = TokenStream::new(); for ridx in grm.iter_rules() { if !grm.rule_to_prods(ridx).contains(&grm.start_prod()) { let r_const = format_ident!("R_{}", grm.rule_name_str(ridx).to_ascii_uppercase()); - let storage_ty = str::parse::(type_name::())?; + let storage_ty = str::parse::(type_name::())?; let ridx = UnsuffixedUsize(usize::from(ridx)); toks.extend(quote! { #[allow(dead_code)] @@ -1179,14 +1177,15 @@ where fn gen_token_epp( &self, - grm: &YaccGrammar, + code_gen: &ParserCodegen, ) -> Result { + let grm = code_gen.grm(); let mut tidxs = Vec::new(); for tidx in grm.iter_tidxs() { tidxs.push(QuoteOption(grm.token_epp(tidx))); } let const_epp_ident = format_ident!("{}EPP", GLOBAL_PREFIX); - let storage_ty = str::parse::(type_name::())?; + let storage_ty = str::parse::(type_name::())?; Ok(quote! { const #const_epp_ident: &[::std::option::Option<&str>] = &[ #(#tidxs,)* @@ -1200,9 +1199,13 @@ where } }) } - /// Generate the wrappers that call user actions - fn gen_wrappers(&self, grm: &YaccGrammar) -> Result> { + fn gen_wrappers( + &self, + code_gen: &ParserCodegen, + build_env: &ParserBuildEnv, + ) -> Result> { + let grm = code_gen.grm(); let parsed_parse_generics = make_generics(grm.parse_generics().as_deref())?; let (generics, type_generics, where_clause) = parsed_parse_generics.split_for_impl(); @@ -1231,10 +1234,10 @@ where let lexer_var = format_ident!("{}lexer", ACTION_PREFIX); let span_var = format_ident!("{}span", ACTION_PREFIX); let args_var = format_ident!("{}args", ACTION_PREFIX); - let storaget = str::parse::(type_name::())?; + let storaget = str::parse::(type_name::())?; let lexertypest = str::parse::(type_name::())?; let actionskind = str::parse::(ACTIONS_KIND)?; - let edition_lifetime = if self.rust_edition != RustEdition::Rust2015 { + let edition_lifetime = if build_env.rust_edition() != RustEdition::Rust2015 { Some(quote!('_,)) } else { None @@ -1363,9 +1366,11 @@ where /// Generate the user action functions (if any). fn gen_user_actions( &self, - grm: &YaccGrammar, - diag: &SpannedDiagnosticFormatter, + code_gen: &ParserCodegen, + src_env: &ParserSrcEnv, ) -> Result> { + let grm = code_gen.grm(); + let diag = src_env.yacc_diag(); let programs = grm .programs() .as_ref() @@ -1440,7 +1445,7 @@ where let lexer_var = format_ident!("{}lexer", ACTION_PREFIX); let span_var = format_ident!("{}span", ACTION_PREFIX); let ridx_var = format_ident!("{}ridx", ACTION_PREFIX); - let storaget = str::parse::(type_name::())?; + let storaget = str::parse::(type_name::())?; let lexertypest = str::parse::(type_name::())?; let bind_parse_param = if !parse_param_unit { Some(quote! {let _ = #parse_paramname;}) @@ -1532,7 +1537,11 @@ where /// Return the `RIdx` of the %start rule in the grammar (which will not be the same as /// grm.start_rule_idx because the latter has an additional rule insert by cfgrammar /// which then calls the user's %start rule). - fn user_start_ridx(&self, grm: &YaccGrammar) -> RIdx { + fn user_start_ridx( + &self, + code_gen: &ParserCodegen, + ) -> RIdx { + let grm = code_gen.grm(); debug_assert_eq!(grm.prod(grm.start_prod()).len(), 1); match grm.prod(grm.start_prod())[0] { Symbol::Rule(ridx) => ridx, From 1c6e6acd3bc83e316858bca170bb8f9cd3215255 Mon Sep 17 00:00:00 2001 From: matt rice Date: Thu, 6 Aug 2026 19:33:04 -0700 Subject: [PATCH 06/14] Move `gen_user_actions` to the codegen module --- lrpar/src/lib/codegen.rs | 187 ++++++++++++++++++++++++++++++++++++- lrpar/src/lib/ctbuilder.rs | 179 +---------------------------------- 2 files changed, 187 insertions(+), 179 deletions(-) diff --git a/lrpar/src/lib/codegen.rs b/lrpar/src/lib/codegen.rs index 7d340c3d1..b9e018722 100644 --- a/lrpar/src/lib/codegen.rs +++ b/lrpar/src/lib/codegen.rs @@ -1,23 +1,32 @@ #![deny(unfulfilled_lint_expectations)] #![expect(dead_code)] -use std::{error::Error, fmt, hash::Hash, marker::PhantomData, path::Path}; +use std::{ + any::type_name, + error::Error, + fmt::{self, Write}, + hash::Hash, + marker::PhantomData, + path::Path, +}; use crate::{ LexerTypes, RecoveryKind, RustEdition, SerialisationFormat, Visibility, - ctbuilder::{CTConflictsError, ERROR, FixIntConfig, VarIntConfig, indent}, + ctbuilder::{ + ACTION_PREFIX, CTConflictsError, ERROR, FixIntConfig, VarIntConfig, indent, make_generics, + }, diagnostics::{DiagnosticFormatter, SpannedDiagnosticFormatter}, }; use cfgrammar::{ - Location, Span, + Location, Span, Symbol, header::{GrmtoolsSectionParser, Header, HeaderValue}, yacc::{YaccGrammar, YaccKind, ast::ASTWithValidityInfo}, }; use lrtable::{Minimiser, StateGraph, StateTable, from_yacc}; use proc_macro2::TokenStream; -use quote::{ToTokens, TokenStreamExt, quote}; +use quote::{ToTokens, TokenStreamExt, format_ident, quote}; use wincode::SchemaWrite; pub(crate) struct ParserSrcEnv<'a> { @@ -484,6 +493,176 @@ where ) -> String { self.gen_cache(src_env, build_env).to_string() } + + /// Generate the user action functions (if any). + pub(crate) fn gen_user_actions( + &self, + src_env: &ParserSrcEnv, + ) -> Result> { + let grm = self.grm(); + let diag = src_env.yacc_diag(); + let programs = grm + .programs() + .as_ref() + .map(|s| str::parse::(s)) + .transpose()?; + let mut action_fns = TokenStream::new(); + // Convert actions to functions + let parsed_parse_generics = make_generics(grm.parse_generics().as_deref())?; + let (generics, _, where_clause) = parsed_parse_generics.split_for_impl(); + let (parse_paramname, parse_paramdef, parse_param_unit); + match grm.parse_param() { + Some((name, tyname)) => { + parse_param_unit = tyname.trim() == "()"; + parse_paramname = str::parse::(name)?; + let ty = str::parse::(tyname)?; + parse_paramdef = quote!(#parse_paramname: #ty); + } + None => { + parse_param_unit = true; + parse_paramname = quote!(()); + parse_paramdef = quote! {_: ()}; + } + }; + for pidx in grm.iter_pidxs() { + if pidx == grm.start_prod() { + continue; + } + + // Work out the right type for each argument + let mut args = Vec::with_capacity(grm.prod(pidx).len()); + for i in 0..grm.prod(pidx).len() { + let argt = match grm.prod(pidx)[i] { + Symbol::Rule(ref_ridx) => { + if let Some(action_type) = grm.actiontype(ref_ridx).as_ref() { + str::parse::(action_type)? + } else { + let mut s = String::from("\n"); + let rule_span = grm.rule_name_span(ref_ridx); + s.push_str(&diag.file_location_msg("Error", Some(rule_span))); + s.push('\n'); + s.push_str(&diag.underline_span_with_text( + rule_span, + "Rule missing action type".to_string(), + '^', + )); + return Err(ErrorString(s).into()); + } + } + Symbol::Token(_) => { + let lexemet = + str::parse::(type_name::())?; + quote!(::std::result::Result<#lexemet, #lexemet>) + } + }; + let arg = format_ident!("{}arg_{}", ACTION_PREFIX, i + 1); + args.push(quote!(mut #arg: #argt)); + } + + // If this rule's `actiont` is `()` then Clippy will warn that the return type `-> ()` + // is pointless (which is true). We therefore avoid outputting a return type if actiont + // is the unit type. + let returnt = { + let actiont = grm.actiontype(grm.prod_to_rule(pidx)).as_ref().unwrap(); + if actiont == "()" { + None + } else { + let actiont = str::parse::(actiont)?; + Some(quote!( -> #actiont)) + } + }; + let action_fn = format_ident!("{}action_{}", ACTION_PREFIX, usize::from(pidx)); + let lexer_var = format_ident!("{}lexer", ACTION_PREFIX); + let span_var = format_ident!("{}span", ACTION_PREFIX); + let ridx_var = format_ident!("{}ridx", ACTION_PREFIX); + let storaget = str::parse::(type_name::())?; + let lexertypest = str::parse::(type_name::())?; + let bind_parse_param = if !parse_param_unit { + Some(quote! {let _ = #parse_paramname;}) + } else { + None + }; + + // Iterate over all $-arguments and replace them with their respective + // element from the argument vector (e.g. $1 is replaced by args[0]). + let pre_action = grm.action(pidx).as_ref().ok_or_else(|| { + let mut s = String::from("\n"); + let span = grm.prod_span(pidx); + s.push_str(&diag.file_location_msg("Error", Some(span))); + s.push('\n'); + s.push_str(&diag.underline_span_with_text( + span, + "Production is missing action code".to_string(), + '^', + )); + ErrorString(s) + })?; + let mut last = 0; + let mut outs = String::new(); + loop { + match pre_action[last..].find('$') { + Some(off) => { + if pre_action[last + off..].starts_with("$$") { + outs.push_str(&pre_action[last..last + off + "$".len()]); + last = last + off + "$$".len(); + } else if pre_action[last + off..].starts_with("$lexer") { + outs.push_str(&pre_action[last..last + off]); + write!(outs, "{prefix}lexer", prefix = ACTION_PREFIX).ok(); + last = last + off + "$lexer".len(); + } else if pre_action[last + off..].starts_with("$span") { + outs.push_str(&pre_action[last..last + off]); + write!(outs, "{prefix}span", prefix = ACTION_PREFIX).ok(); + last = last + off + "$span".len(); + } else if last + off + 1 < pre_action.len() + && pre_action[last + off + 1..].starts_with(|c: char| c.is_numeric()) + { + outs.push_str(&pre_action[last..last + off]); + write!(outs, "{prefix}arg_", prefix = ACTION_PREFIX).ok(); + last = last + off + "$".len(); + } else { + let span = grm.action_span(pidx).unwrap(); + let inner_span = + Span::new(span.start() + last + off + "$".len(), span.end()); + let mut s = String::from("\n"); + s.push_str(&diag.file_location_msg("Error", Some(inner_span))); + s.push('\n'); + s.push_str(&diag.underline_span_with_text( + inner_span, + "Unknown text following '$'".to_string(), + '^', + )); + return Err(ErrorString(s).into()); + } + } + None => { + outs.push_str(&pre_action[last..]); + break; + } + } + } + + let action_body = str::parse::(&outs)?; + action_fns.extend(quote! { + #[allow(clippy::too_many_arguments)] + fn #action_fn #generics ( + #ridx_var: ::cfgrammar::RIdx<#storaget>, + #lexer_var: &'lexer dyn ::lrpar::NonStreamingLexer<'input, #lexertypest>, + #span_var: ::cfgrammar::Span, + #parse_paramdef, + #(#args,)* + ) #returnt + #where_clause + { + #bind_parse_param + #action_body + } + }) + } + Ok(quote! { + #programs + #action_fns + }) + } } /// A string which uses `Display` for it's `Debug` impl. diff --git a/lrpar/src/lib/ctbuilder.rs b/lrpar/src/lib/ctbuilder.rs index 0bf9d1fec..535d8683e 100644 --- a/lrpar/src/lib/ctbuilder.rs +++ b/lrpar/src/lib/ctbuilder.rs @@ -24,7 +24,7 @@ use crate::{ use crate::unstable_api::UnstableApi; use cfgrammar::{ - Location, RIdx, Span, Symbol, + Location, RIdx, Symbol, header::{Header, HeaderError, HeaderErrorKind, HeaderValue, Namespaced, Setting, Value}, markmap::{Entry, MergeBehavior}, yacc::{YaccGrammar, YaccKind, YaccOriginalActionKind, ast::ASTWithValidityInfo}, @@ -37,7 +37,7 @@ use quote::{ToTokens, TokenStreamExt, format_ident, quote}; use syn::{Generics, parse_quote}; use wincode::{SchemaRead, SchemaReadOwned, SchemaWrite}; -const ACTION_PREFIX: &str = "__gt_"; +pub(crate) const ACTION_PREFIX: &str = "__gt_"; const GLOBAL_PREFIX: &str = "__GT_"; const ACTIONS_KIND: &str = "__GtActionsKind"; const ACTIONS_KIND_PREFIX: &str = "Ak"; @@ -904,7 +904,7 @@ where let user_actions = if let YaccKind::Original(YaccOriginalActionKind::UserAction) | YaccKind::Grmtools = build_env.yacc_kind() { - Some(self.gen_user_actions(code_gen, src_env)?) + Some(code_gen.gen_user_actions(src_env)?) } else { None }; @@ -1363,177 +1363,6 @@ where Ok(wrappers) } - /// Generate the user action functions (if any). - fn gen_user_actions( - &self, - code_gen: &ParserCodegen, - src_env: &ParserSrcEnv, - ) -> Result> { - let grm = code_gen.grm(); - let diag = src_env.yacc_diag(); - let programs = grm - .programs() - .as_ref() - .map(|s| str::parse::(s)) - .transpose()?; - let mut action_fns = TokenStream::new(); - // Convert actions to functions - let parsed_parse_generics = make_generics(grm.parse_generics().as_deref())?; - let (generics, _, where_clause) = parsed_parse_generics.split_for_impl(); - let (parse_paramname, parse_paramdef, parse_param_unit); - match grm.parse_param() { - Some((name, tyname)) => { - parse_param_unit = tyname.trim() == "()"; - parse_paramname = str::parse::(name)?; - let ty = str::parse::(tyname)?; - parse_paramdef = quote!(#parse_paramname: #ty); - } - None => { - parse_param_unit = true; - parse_paramname = quote!(()); - parse_paramdef = quote! {_: ()}; - } - }; - for pidx in grm.iter_pidxs() { - if pidx == grm.start_prod() { - continue; - } - - // Work out the right type for each argument - let mut args = Vec::with_capacity(grm.prod(pidx).len()); - for i in 0..grm.prod(pidx).len() { - let argt = match grm.prod(pidx)[i] { - Symbol::Rule(ref_ridx) => { - if let Some(action_type) = grm.actiontype(ref_ridx).as_ref() { - str::parse::(action_type)? - } else { - let mut s = String::from("\n"); - let rule_span = grm.rule_name_span(ref_ridx); - s.push_str(&diag.file_location_msg("Error", Some(rule_span))); - s.push('\n'); - s.push_str(&diag.underline_span_with_text( - rule_span, - "Rule missing action type".to_string(), - '^', - )); - return Err(ErrorString(s).into()); - } - } - Symbol::Token(_) => { - let lexemet = - str::parse::(type_name::())?; - quote!(::std::result::Result<#lexemet, #lexemet>) - } - }; - let arg = format_ident!("{}arg_{}", ACTION_PREFIX, i + 1); - args.push(quote!(mut #arg: #argt)); - } - - // If this rule's `actiont` is `()` then Clippy will warn that the return type `-> ()` - // is pointless (which is true). We therefore avoid outputting a return type if actiont - // is the unit type. - let returnt = { - let actiont = grm.actiontype(grm.prod_to_rule(pidx)).as_ref().unwrap(); - if actiont == "()" { - None - } else { - let actiont = str::parse::(actiont)?; - Some(quote!( -> #actiont)) - } - }; - let action_fn = format_ident!("{}action_{}", ACTION_PREFIX, usize::from(pidx)); - let lexer_var = format_ident!("{}lexer", ACTION_PREFIX); - let span_var = format_ident!("{}span", ACTION_PREFIX); - let ridx_var = format_ident!("{}ridx", ACTION_PREFIX); - let storaget = str::parse::(type_name::())?; - let lexertypest = str::parse::(type_name::())?; - let bind_parse_param = if !parse_param_unit { - Some(quote! {let _ = #parse_paramname;}) - } else { - None - }; - - // Iterate over all $-arguments and replace them with their respective - // element from the argument vector (e.g. $1 is replaced by args[0]). - let pre_action = grm.action(pidx).as_ref().ok_or_else(|| { - let mut s = String::from("\n"); - let span = grm.prod_span(pidx); - s.push_str(&diag.file_location_msg("Error", Some(span))); - s.push('\n'); - s.push_str(&diag.underline_span_with_text( - span, - "Production is missing action code".to_string(), - '^', - )); - ErrorString(s) - })?; - let mut last = 0; - let mut outs = String::new(); - loop { - match pre_action[last..].find('$') { - Some(off) => { - if pre_action[last + off..].starts_with("$$") { - outs.push_str(&pre_action[last..last + off + "$".len()]); - last = last + off + "$$".len(); - } else if pre_action[last + off..].starts_with("$lexer") { - outs.push_str(&pre_action[last..last + off]); - write!(outs, "{prefix}lexer", prefix = ACTION_PREFIX).ok(); - last = last + off + "$lexer".len(); - } else if pre_action[last + off..].starts_with("$span") { - outs.push_str(&pre_action[last..last + off]); - write!(outs, "{prefix}span", prefix = ACTION_PREFIX).ok(); - last = last + off + "$span".len(); - } else if last + off + 1 < pre_action.len() - && pre_action[last + off + 1..].starts_with(|c: char| c.is_numeric()) - { - outs.push_str(&pre_action[last..last + off]); - write!(outs, "{prefix}arg_", prefix = ACTION_PREFIX).ok(); - last = last + off + "$".len(); - } else { - let span = grm.action_span(pidx).unwrap(); - let inner_span = - Span::new(span.start() + last + off + "$".len(), span.end()); - let mut s = String::from("\n"); - s.push_str(&diag.file_location_msg("Error", Some(inner_span))); - s.push('\n'); - s.push_str(&diag.underline_span_with_text( - inner_span, - "Unknown text following '$'".to_string(), - '^', - )); - return Err(ErrorString(s).into()); - } - } - None => { - outs.push_str(&pre_action[last..]); - break; - } - } - } - - let action_body = str::parse::(&outs)?; - action_fns.extend(quote! { - #[allow(clippy::too_many_arguments)] - fn #action_fn #generics ( - #ridx_var: ::cfgrammar::RIdx<#storaget>, - #lexer_var: &'lexer dyn ::lrpar::NonStreamingLexer<'input, #lexertypest>, - #span_var: ::cfgrammar::Span, - #parse_paramdef, - #(#args,)* - ) #returnt - #where_clause - { - #bind_parse_param - #action_body - } - }) - } - Ok(quote! { - #programs - #action_fns - }) - } - /// Return the `RIdx` of the %start rule in the grammar (which will not be the same as /// grm.start_rule_idx because the latter has an additional rule insert by cfgrammar /// which then calls the user's %start rule). @@ -1667,7 +1496,7 @@ pub(crate) fn indent(indent: &str, s: &str) -> String { format!("{indent}{}\n", s.trim_end_matches('\n')).replace('\n', &format!("\n{}", indent)) } -fn make_generics(parse_generics: Option<&str>) -> Result> { +pub(crate) fn make_generics(parse_generics: Option<&str>) -> Result> { if let Some(parse_generics) = parse_generics { let tokens = str::parse::(parse_generics)?; match syn::parse2(quote!(<'lexer, 'input: 'lexer, #tokens>)) { From 9690e116e3ddbd1b156b008721c48dba666919b5 Mon Sep 17 00:00:00 2001 From: matt rice Date: Thu, 6 Aug 2026 19:38:25 -0700 Subject: [PATCH 07/14] Move `gen_rule_consts` to the codegen module --- lrpar/src/lib/codegen.rs | 31 ++++++++++++++++++++++++++++++- lrpar/src/lib/ctbuilder.rs | 38 +++----------------------------------- 2 files changed, 33 insertions(+), 36 deletions(-) diff --git a/lrpar/src/lib/codegen.rs b/lrpar/src/lib/codegen.rs index b9e018722..a8f08de93 100644 --- a/lrpar/src/lib/codegen.rs +++ b/lrpar/src/lib/codegen.rs @@ -25,7 +25,7 @@ use cfgrammar::{ }; use lrtable::{Minimiser, StateGraph, StateTable, from_yacc}; -use proc_macro2::TokenStream; +use proc_macro2::{Literal, TokenStream}; use quote::{ToTokens, TokenStreamExt, format_ident, quote}; use wincode::SchemaWrite; @@ -663,6 +663,23 @@ where #action_fns }) } + + pub(crate) fn gen_rule_consts(&self) -> Result { + let grm = self.grm(); + let mut toks = TokenStream::new(); + for ridx in grm.iter_rules() { + if !grm.rule_to_prods(ridx).contains(&grm.start_prod()) { + let r_const = format_ident!("R_{}", grm.rule_name_str(ridx).to_ascii_uppercase()); + let storage_ty = str::parse::(type_name::())?; + let ridx = UnsuffixedUsize(usize::from(ridx)); + toks.extend(quote! { + #[allow(dead_code)] + pub const #r_const: #storage_ty = #ridx; + }); + } + } + Ok(toks) + } } /// A string which uses `Display` for it's `Debug` impl. @@ -719,6 +736,18 @@ impl ToTokens for QuoteToString<'_> { } } +/// The quote impl of `ToTokens` for `usize` prints literal values +/// including a type suffix for example `0usize`. +/// +/// This wrapper omits the type suffix emitting `0` instead. +struct UnsuffixedUsize(usize); + +impl ToTokens for UnsuffixedUsize { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.append(Literal::usize_unsuffixed(self.0)) + } +} + impl RustEdition { fn to_variant_tokens(self) -> TokenStream { match self { diff --git a/lrpar/src/lib/ctbuilder.rs b/lrpar/src/lib/ctbuilder.rs index 535d8683e..d99219848 100644 --- a/lrpar/src/lib/ctbuilder.rs +++ b/lrpar/src/lib/ctbuilder.rs @@ -32,8 +32,8 @@ use cfgrammar::{ use filetime::FileTime; use lrtable::{StateGraph, StateTable, statetable::Conflicts}; use num_traits::{AsPrimitive, PrimInt, Unsigned}; -use proc_macro2::{Literal, TokenStream}; -use quote::{ToTokens, TokenStreamExt, format_ident, quote}; +use proc_macro2::TokenStream; +use quote::{ToTokens, format_ident, quote}; use syn::{Generics, parse_quote}; use wincode::{SchemaRead, SchemaReadOwned, SchemaWrite}; @@ -59,18 +59,6 @@ pub(crate) struct CTConflictsError { pub(crate) phantom: PhantomData, } -/// The quote impl of `ToTokens` for `usize` prints literal values -/// including a type suffix for example `0usize`. -/// -/// This wrapper omits the type suffix emitting `0` instead. -struct UnsuffixedUsize(usize); - -impl ToTokens for UnsuffixedUsize { - fn to_tokens(&self, tokens: &mut TokenStream) { - tokens.append(Literal::usize_unsuffixed(self.0)) - } -} - impl fmt::Display for CTConflictsError where StorageT: 'static + Debug + Hash + PrimInt + Unsigned, @@ -909,7 +897,7 @@ where None }; - let rule_consts = self.gen_rule_consts(code_gen)?; + let rule_consts = code_gen.gen_rule_consts()?; let token_epp = self.gen_token_epp(code_gen)?; let parse_function = self.gen_parse_function(code_gen, build_env)?; let action_wrappers = match build_env.yacc_kind() { @@ -1155,26 +1143,6 @@ where }) } - fn gen_rule_consts( - &self, - code_gen: &ParserCodegen, - ) -> Result { - let grm = code_gen.grm(); - let mut toks = TokenStream::new(); - for ridx in grm.iter_rules() { - if !grm.rule_to_prods(ridx).contains(&grm.start_prod()) { - let r_const = format_ident!("R_{}", grm.rule_name_str(ridx).to_ascii_uppercase()); - let storage_ty = str::parse::(type_name::())?; - let ridx = UnsuffixedUsize(usize::from(ridx)); - toks.extend(quote! { - #[allow(dead_code)] - pub const #r_const: #storage_ty = #ridx; - }); - } - } - Ok(toks) - } - fn gen_token_epp( &self, code_gen: &ParserCodegen, From 7c7f428bca78bc1a9bfcd9bf873f049228c73de4 Mon Sep 17 00:00:00 2001 From: matt rice Date: Thu, 6 Aug 2026 19:42:21 -0700 Subject: [PATCH 08/14] Move `gen_token_epp` to codegen module --- lrpar/src/lib/codegen.rs | 28 +++++++++++++++++++++++++--- lrpar/src/lib/ctbuilder.rs | 30 +++--------------------------- 2 files changed, 28 insertions(+), 30 deletions(-) diff --git a/lrpar/src/lib/codegen.rs b/lrpar/src/lib/codegen.rs index a8f08de93..72da0235d 100644 --- a/lrpar/src/lib/codegen.rs +++ b/lrpar/src/lib/codegen.rs @@ -13,7 +13,8 @@ use std::{ use crate::{ LexerTypes, RecoveryKind, RustEdition, SerialisationFormat, Visibility, ctbuilder::{ - ACTION_PREFIX, CTConflictsError, ERROR, FixIntConfig, VarIntConfig, indent, make_generics, + ACTION_PREFIX, CTConflictsError, ERROR, FixIntConfig, GLOBAL_PREFIX, VarIntConfig, indent, + make_generics, }, diagnostics::{DiagnosticFormatter, SpannedDiagnosticFormatter}, }; @@ -680,6 +681,28 @@ where } Ok(toks) } + + pub(crate) fn gen_token_epp(&self) -> Result { + let grm = self.grm(); + let mut tidxs = Vec::new(); + for tidx in grm.iter_tidxs() { + tidxs.push(QuoteOption(grm.token_epp(tidx))); + } + let const_epp_ident = format_ident!("{}EPP", GLOBAL_PREFIX); + let storage_ty = str::parse::(type_name::())?; + Ok(quote! { + const #const_epp_ident: &[::std::option::Option<&str>] = &[ + #(#tidxs,)* + ]; + + /// Return the %epp entry for token `tidx` (where `None` indicates \"the token has no + /// pretty-printed value\"). Panics if `tidx` doesn't exist. + #[allow(dead_code)] + pub fn token_epp<'a>(tidx: ::cfgrammar::TIdx<#storage_ty>) -> ::std::option::Option<&'a str> { + #const_epp_ident[usize::from(tidx)] + } + }) + } } /// A string which uses `Display` for it's `Debug` impl. @@ -703,8 +726,7 @@ impl Error for ErrorString {} /// /// This wrapper instead emits both `Some` and `None` variants. /// See: [quote #20](https://github.com/dtolnay/quote/issues/20) -// Temporarily pub(crate) -pub(crate) struct QuoteOption(pub(crate) Option); +struct QuoteOption(Option); impl ToTokens for QuoteOption { fn to_tokens(&self, tokens: &mut TokenStream) { diff --git a/lrpar/src/lib/ctbuilder.rs b/lrpar/src/lib/ctbuilder.rs index d99219848..82cbd4220 100644 --- a/lrpar/src/lib/ctbuilder.rs +++ b/lrpar/src/lib/ctbuilder.rs @@ -16,7 +16,7 @@ use std::{ use crate::{ LexerTypes, RTParserBuilder, RecoveryKind, - codegen::{ParserBuildEnv, ParserBuildEnvArgs, ParserCodegen, ParserSrcEnv, QuoteOption}, + codegen::{ParserBuildEnv, ParserBuildEnvArgs, ParserCodegen, ParserSrcEnv}, diagnostics::DiagnosticFormatter, }; @@ -38,7 +38,7 @@ use syn::{Generics, parse_quote}; use wincode::{SchemaRead, SchemaReadOwned, SchemaWrite}; pub(crate) const ACTION_PREFIX: &str = "__gt_"; -const GLOBAL_PREFIX: &str = "__GT_"; +pub(crate) const GLOBAL_PREFIX: &str = "__GT_"; const ACTIONS_KIND: &str = "__GtActionsKind"; const ACTIONS_KIND_PREFIX: &str = "Ak"; const ACTIONS_KIND_HIDDEN: &str = "__GtActionsKindHidden"; @@ -898,7 +898,7 @@ where }; let rule_consts = code_gen.gen_rule_consts()?; - let token_epp = self.gen_token_epp(code_gen)?; + let token_epp = code_gen.gen_token_epp()?; let parse_function = self.gen_parse_function(code_gen, build_env)?; let action_wrappers = match build_env.yacc_kind() { YaccKind::Original(YaccOriginalActionKind::UserAction) | YaccKind::Grmtools => { @@ -1143,30 +1143,6 @@ where }) } - fn gen_token_epp( - &self, - code_gen: &ParserCodegen, - ) -> Result { - let grm = code_gen.grm(); - let mut tidxs = Vec::new(); - for tidx in grm.iter_tidxs() { - tidxs.push(QuoteOption(grm.token_epp(tidx))); - } - let const_epp_ident = format_ident!("{}EPP", GLOBAL_PREFIX); - let storage_ty = str::parse::(type_name::())?; - Ok(quote! { - const #const_epp_ident: &[::std::option::Option<&str>] = &[ - #(#tidxs,)* - ]; - - /// Return the %epp entry for token `tidx` (where `None` indicates \"the token has no - /// pretty-printed value\"). Panics if `tidx` doesn't exist. - #[allow(dead_code)] - pub fn token_epp<'a>(tidx: ::cfgrammar::TIdx<#storage_ty>) -> ::std::option::Option<&'a str> { - #const_epp_ident[usize::from(tidx)] - } - }) - } /// Generate the wrappers that call user actions fn gen_wrappers( &self, From 2d95ae453312b6587e861b93943c56866a193924 Mon Sep 17 00:00:00 2001 From: matt rice Date: Thu, 6 Aug 2026 19:50:08 -0700 Subject: [PATCH 09/14] Move `gen_parse_function` to the codegen module --- lrpar/src/lib/codegen.rs | 199 ++++++++++++++++++++++++++++++++++- lrpar/src/lib/ctbuilder.rs | 206 ++----------------------------------- 2 files changed, 201 insertions(+), 204 deletions(-) diff --git a/lrpar/src/lib/codegen.rs b/lrpar/src/lib/codegen.rs index 72da0235d..2e07d1aac 100644 --- a/lrpar/src/lib/codegen.rs +++ b/lrpar/src/lib/codegen.rs @@ -13,21 +13,22 @@ use std::{ use crate::{ LexerTypes, RecoveryKind, RustEdition, SerialisationFormat, Visibility, ctbuilder::{ - ACTION_PREFIX, CTConflictsError, ERROR, FixIntConfig, GLOBAL_PREFIX, VarIntConfig, indent, - make_generics, + ACTION_PREFIX, ACTIONS_KIND, ACTIONS_KIND_PREFIX, CTConflictsError, ERROR, FixIntConfig, + GLOBAL_PREFIX, VarIntConfig, indent, make_generics, }, diagnostics::{DiagnosticFormatter, SpannedDiagnosticFormatter}, }; use cfgrammar::{ - Location, Span, Symbol, + Location, RIdx, Span, Symbol, header::{GrmtoolsSectionParser, Header, HeaderValue}, - yacc::{YaccGrammar, YaccKind, ast::ASTWithValidityInfo}, + yacc::{YaccGrammar, YaccKind, YaccOriginalActionKind, ast::ASTWithValidityInfo}, }; use lrtable::{Minimiser, StateGraph, StateTable, from_yacc}; use proc_macro2::{Literal, TokenStream}; use quote::{ToTokens, TokenStreamExt, format_ident, quote}; +use syn::Generics; use wincode::SchemaWrite; pub(crate) struct ParserSrcEnv<'a> { @@ -703,6 +704,196 @@ where } }) } + + /// Generate the main parse() function for the output file. + pub(crate) fn gen_parse_function( + &self, + build_env: &ParserBuildEnv, + ) -> Result> { + let stable = self.stable(); + let grm = self.grm(); + let storaget = str::parse::(type_name::())?; + let lexertypest = str::parse::(type_name::())?; + let recoverer = build_env.recoverer(); + let run_parser = match build_env.ast_validation().yacc_kind() { + YaccKind::Original(YaccOriginalActionKind::GenericParseTree) => { + quote! { + ::lrpar::RTParserBuilder::new(grm, stable) + .recoverer(#recoverer) + .parse_map( + lexer, + &|lexeme| Node::Term{lexeme}, + &|ridx, nodes| Node::Nonterm{ridx, nodes} + ) + } + } + YaccKind::Original(YaccOriginalActionKind::NoAction) => { + quote! { + ::lrpar::RTParserBuilder::new(grm, stable) + .recoverer(#recoverer) + .parse_map(lexer, &|_| (), &|_, _| ()).1 + } + } + YaccKind::Original(YaccOriginalActionKind::UserAction) | YaccKind::Grmtools => { + let actionskind = str::parse::(ACTIONS_KIND)?; + let parsed_parse_generics = make_generics(grm.parse_generics().as_deref())?; + let (_, type_generics, _) = parsed_parse_generics.split_for_impl(); + // actions always have a parse_param argument, and when the `parse` function lacks one + // that parameter will be unit. + let (action_fn_parse_param, action_fn_parse_param_ty) = match grm.parse_param() { + Some((name, ty)) => { + let name = str::parse::(name)?; + let ty = str::parse::(ty)?; + (quote!(#name), quote!(#ty)) + } + None => (quote!(()), quote!(())), + }; + let wrappers = grm.iter_pidxs().map(|pidx| { + let pidx = usize::from(pidx); + format_ident!("{}wrapper_{}", ACTION_PREFIX, pidx) + }); + let edition_lifetime = if build_env.rust_edition() != RustEdition::Rust2015 { + quote!('_,) + } else { + quote!() + }; + let ridx = usize::from(self.user_start_ridx()); + let action_ident = format_ident!("{}{}", ACTIONS_KIND_PREFIX, ridx); + + quote! { + let actions: ::std::vec::Vec< + &dyn Fn( + ::cfgrammar::RIdx<#storaget>, + &'lexer dyn ::lrpar::NonStreamingLexer<'input, #lexertypest>, + ::cfgrammar::Span, + ::std::vec::Drain<#edition_lifetime ::lrpar::parser::AStackType<<#lexertypest as ::lrpar::LexerTypes>::LexemeT, #actionskind #type_generics>>, + #action_fn_parse_param_ty + ) -> #actionskind #type_generics + > = ::std::vec![#(&#wrappers,)*]; + match ::lrpar::RTParserBuilder::new(grm, stable) + .recoverer(#recoverer) + .parse_actions(lexer, &actions, #action_fn_parse_param) { + (Some(#actionskind::#action_ident(x)), y) => (Some(x), y), + (None, y) => (None, y), + _ => unreachable!() + } + } + } + kind => panic!("YaccKind {:?} not supported", kind), + }; + + let parsed_parse_generics: Generics = match build_env.ast_validation().yacc_kind() { + YaccKind::Original(YaccOriginalActionKind::UserAction) | YaccKind::Grmtools => { + make_generics(grm.parse_generics().as_deref())? + } + _ => make_generics(None)?, + }; + let (generics, _, where_clause) = parsed_parse_generics.split_for_impl(); + + // `parse()` may or may not have an argument for `%parseparam`. + let parse_fn_parse_param = match build_env.ast_validation().yacc_kind() { + YaccKind::Original(YaccOriginalActionKind::UserAction) | YaccKind::Grmtools => { + if let Some((name, tyname)) = grm.parse_param() { + let name = str::parse::(name)?; + let tyname = str::parse::(tyname)?; + Some(quote! {#name: #tyname}) + } else { + None + } + } + _ => None, + }; + let parse_fn_return_ty = match build_env.ast_validation().yacc_kind() { + YaccKind::Original(YaccOriginalActionKind::UserAction) | YaccKind::Grmtools => { + let actiont = grm + .actiontype(self.user_start_ridx()) + .as_ref() + .map(|at| str::parse::(at)) + .transpose()?; + quote! { + (::std::option::Option<#actiont>, ::std::vec::Vec<::lrpar::LexParseError<#storaget, #lexertypest>>) + } + } + YaccKind::Original(YaccOriginalActionKind::GenericParseTree) => quote! { + (::std::option::Option::LexemeT, #storaget>>, + ::std::vec::Vec<::lrpar::LexParseError<#storaget, #lexertypest>>) + }, + YaccKind::Original(YaccOriginalActionKind::NoAction) => quote! { + ::std::vec::Vec<::lrpar::LexParseError<#storaget, #lexertypest>> + }, + _ => unreachable!(), + }; + + let serialisation_format = build_env.serialisation_format(); + // Note that the configuration types use associated consts, and thus these configurations represent distinct types. + let (grm_data, stable_data): (Vec, Vec) = match serialisation_format { + SerialisationFormat::FixedSizeInteger => { + let config = wincode::config::Configuration::default().with_fixint_encoding(); + let grm = wincode::config::serialize(grm, config)?; + let stable = wincode::config::serialize(stable, config)?; + (grm, stable) + } + SerialisationFormat::VariableSizedInteger => { + let config = wincode::config::Configuration::default().with_varint_encoding(); + let grm = wincode::config::serialize(grm, config)?; + let stable = wincode::config::serialize(stable, config)?; + (grm, stable) + } + }; + let serialisation_format_str = quote!(serialisation_format).to_string(); + Ok(quote! { + const __GRM_DATA: &[u8] = &[#(#grm_data,)*]; + const __STABLE_DATA: &[u8] = &[#(#stable_data,)*]; + const __SERIALISATION_FORMAT: ::lrpar::ctbuilder::SerialisationFormat = #serialisation_format; + + fn __lrpar_parser_data() -> &'static ::lrpar::ParserData<#storaget> { + static DATA: ::std::sync::OnceLock<::lrpar::ParserData<#storaget>> + = ::std::sync::OnceLock::new(); + DATA.get_or_init( + || { + // We have to call reconstitute like this because the config parameter takes a trait + // which uses const generics. Thus the two config parameters here are not actually of the same type. + match __SERIALISATION_FORMAT { + ::lrpar::ctbuilder::SerialisationFormat::FixedSizeInteger => { + ::lrpar::ctbuilder::_reconstitute(__GRM_DATA, __STABLE_DATA, ::lrpar::ctbuilder::wincode::config::Configuration::default().with_fixint_encoding()) + } + ::lrpar::ctbuilder::SerialisationFormat::VariableSizedInteger => { + ::lrpar::ctbuilder::_reconstitute(__GRM_DATA, __STABLE_DATA, ::lrpar::ctbuilder::wincode::config::Configuration::default().with_varint_encoding()) + } + _ => { + panic!("Parser source was generated using unknown `SerialisationFormat`: {:?}", #serialisation_format_str) + } + } + } + ) + } + + #[allow(dead_code)] + pub fn parse #generics ( + lexer: &'lexer dyn ::lrpar::NonStreamingLexer<'input, #lexertypest>, + #parse_fn_parse_param + ) -> #parse_fn_return_ty + #where_clause + { + let __data = __lrpar_parser_data(); + let grm = __data.grm(); + let stable = __data.stable(); + #run_parser + } + }) + } + + /// Return the `RIdx` of the %start rule in the grammar (which will not be the same as + /// grm.start_rule_idx because the latter has an additional rule insert by cfgrammar + /// which then calls the user's %start rule). + fn user_start_ridx(&self) -> RIdx { + let grm = self.grm(); + debug_assert_eq!(grm.prod(grm.start_prod()).len(), 1); + match grm.prod(grm.start_prod())[0] { + Symbol::Rule(ridx) => ridx, + _ => unreachable!(), + } + } } /// A string which uses `Display` for it's `Debug` impl. diff --git a/lrpar/src/lib/ctbuilder.rs b/lrpar/src/lib/ctbuilder.rs index 82cbd4220..15e7e7ac5 100644 --- a/lrpar/src/lib/ctbuilder.rs +++ b/lrpar/src/lib/ctbuilder.rs @@ -24,7 +24,7 @@ use crate::{ use crate::unstable_api::UnstableApi; use cfgrammar::{ - Location, RIdx, Symbol, + Location, Symbol, header::{Header, HeaderError, HeaderErrorKind, HeaderValue, Namespaced, Setting, Value}, markmap::{Entry, MergeBehavior}, yacc::{YaccGrammar, YaccKind, YaccOriginalActionKind, ast::ASTWithValidityInfo}, @@ -39,9 +39,9 @@ use wincode::{SchemaRead, SchemaReadOwned, SchemaWrite}; pub(crate) const ACTION_PREFIX: &str = "__gt_"; pub(crate) const GLOBAL_PREFIX: &str = "__GT_"; -const ACTIONS_KIND: &str = "__GtActionsKind"; -const ACTIONS_KIND_PREFIX: &str = "Ak"; -const ACTIONS_KIND_HIDDEN: &str = "__GtActionsKindHidden"; +pub(crate) const ACTIONS_KIND: &str = "__GtActionsKind"; +pub(crate) const ACTIONS_KIND_PREFIX: &str = "Ak"; +pub(crate) const ACTIONS_KIND_HIDDEN: &str = "__GtActionsKindHidden"; const RUST_FILE_EXT: &str = "rs"; @@ -899,8 +899,8 @@ where let rule_consts = code_gen.gen_rule_consts()?; let token_epp = code_gen.gen_token_epp()?; - let parse_function = self.gen_parse_function(code_gen, build_env)?; - let action_wrappers = match build_env.yacc_kind() { + let parse_function = code_gen.gen_parse_function(build_env)?; + let action_wrappers = match build_env.yacc_kind() { YaccKind::Original(YaccOriginalActionKind::UserAction) | YaccKind::Grmtools => { Some(self.gen_wrappers(code_gen, build_env)?) } @@ -964,185 +964,6 @@ where Ok(()) } - /// Generate the main parse() function for the output file. - fn gen_parse_function( - &self, - code_gen: &ParserCodegen, - build_env: &ParserBuildEnv, - ) -> Result> { - let stable = code_gen.stable(); - let grm = code_gen.grm(); - let storaget = str::parse::(type_name::())?; - let lexertypest = str::parse::(type_name::())?; - let recoverer = build_env.recoverer(); - let run_parser = match build_env.ast_validation().yacc_kind() { - YaccKind::Original(YaccOriginalActionKind::GenericParseTree) => { - quote! { - ::lrpar::RTParserBuilder::new(grm, stable) - .recoverer(#recoverer) - .parse_map( - lexer, - &|lexeme| Node::Term{lexeme}, - &|ridx, nodes| Node::Nonterm{ridx, nodes} - ) - } - } - YaccKind::Original(YaccOriginalActionKind::NoAction) => { - quote! { - ::lrpar::RTParserBuilder::new(grm, stable) - .recoverer(#recoverer) - .parse_map(lexer, &|_| (), &|_, _| ()).1 - } - } - YaccKind::Original(YaccOriginalActionKind::UserAction) | YaccKind::Grmtools => { - let actionskind = str::parse::(ACTIONS_KIND)?; - let parsed_parse_generics = make_generics(grm.parse_generics().as_deref())?; - let (_, type_generics, _) = parsed_parse_generics.split_for_impl(); - // actions always have a parse_param argument, and when the `parse` function lacks one - // that parameter will be unit. - let (action_fn_parse_param, action_fn_parse_param_ty) = match grm.parse_param() { - Some((name, ty)) => { - let name = str::parse::(name)?; - let ty = str::parse::(ty)?; - (quote!(#name), quote!(#ty)) - } - None => (quote!(()), quote!(())), - }; - let wrappers = grm.iter_pidxs().map(|pidx| { - let pidx = usize::from(pidx); - format_ident!("{}wrapper_{}", ACTION_PREFIX, pidx) - }); - let edition_lifetime = if build_env.rust_edition() != RustEdition::Rust2015 { - quote!('_,) - } else { - quote!() - }; - let ridx = usize::from(self.user_start_ridx(code_gen)); - let action_ident = format_ident!("{}{}", ACTIONS_KIND_PREFIX, ridx); - - quote! { - let actions: ::std::vec::Vec< - &dyn Fn( - ::cfgrammar::RIdx<#storaget>, - &'lexer dyn ::lrpar::NonStreamingLexer<'input, #lexertypest>, - ::cfgrammar::Span, - ::std::vec::Drain<#edition_lifetime ::lrpar::parser::AStackType<<#lexertypest as ::lrpar::LexerTypes>::LexemeT, #actionskind #type_generics>>, - #action_fn_parse_param_ty - ) -> #actionskind #type_generics - > = ::std::vec![#(&#wrappers,)*]; - match ::lrpar::RTParserBuilder::new(grm, stable) - .recoverer(#recoverer) - .parse_actions(lexer, &actions, #action_fn_parse_param) { - (Some(#actionskind::#action_ident(x)), y) => (Some(x), y), - (None, y) => (None, y), - _ => unreachable!() - } - } - } - kind => panic!("YaccKind {:?} not supported", kind), - }; - - let parsed_parse_generics: Generics = match build_env.ast_validation().yacc_kind() { - YaccKind::Original(YaccOriginalActionKind::UserAction) | YaccKind::Grmtools => { - make_generics(grm.parse_generics().as_deref())? - } - _ => make_generics(None)?, - }; - let (generics, _, where_clause) = parsed_parse_generics.split_for_impl(); - - // `parse()` may or may not have an argument for `%parseparam`. - let parse_fn_parse_param = match build_env.ast_validation().yacc_kind() { - YaccKind::Original(YaccOriginalActionKind::UserAction) | YaccKind::Grmtools => { - if let Some((name, tyname)) = grm.parse_param() { - let name = str::parse::(name)?; - let tyname = str::parse::(tyname)?; - Some(quote! {#name: #tyname}) - } else { - None - } - } - _ => None, - }; - let parse_fn_return_ty = match build_env.ast_validation().yacc_kind() { - YaccKind::Original(YaccOriginalActionKind::UserAction) | YaccKind::Grmtools => { - let actiont = grm - .actiontype(self.user_start_ridx(code_gen)) - .as_ref() - .map(|at| str::parse::(at)) - .transpose()?; - quote! { - (::std::option::Option<#actiont>, ::std::vec::Vec<::lrpar::LexParseError<#storaget, #lexertypest>>) - } - } - YaccKind::Original(YaccOriginalActionKind::GenericParseTree) => quote! { - (::std::option::Option::LexemeT, #storaget>>, - ::std::vec::Vec<::lrpar::LexParseError<#storaget, #lexertypest>>) - }, - YaccKind::Original(YaccOriginalActionKind::NoAction) => quote! { - ::std::vec::Vec<::lrpar::LexParseError<#storaget, #lexertypest>> - }, - _ => unreachable!(), - }; - - let serialisation_format = build_env.serialisation_format(); - // Note that the configuration types use associated consts, and thus these configurations represent distinct types. - let (grm_data, stable_data): (Vec, Vec) = match serialisation_format { - SerialisationFormat::FixedSizeInteger => { - let config = wincode::config::Configuration::default().with_fixint_encoding(); - let grm = wincode::config::serialize(grm, config)?; - let stable = wincode::config::serialize(stable, config)?; - (grm, stable) - } - SerialisationFormat::VariableSizedInteger => { - let config = wincode::config::Configuration::default().with_varint_encoding(); - let grm = wincode::config::serialize(grm, config)?; - let stable = wincode::config::serialize(stable, config)?; - (grm, stable) - } - }; - let serialisation_format_str = quote!(serialisation_format).to_string(); - Ok(quote! { - const __GRM_DATA: &[u8] = &[#(#grm_data,)*]; - const __STABLE_DATA: &[u8] = &[#(#stable_data,)*]; - const __SERIALISATION_FORMAT: ::lrpar::ctbuilder::SerialisationFormat = #serialisation_format; - - fn __lrpar_parser_data() -> &'static ::lrpar::ParserData<#storaget> { - static DATA: ::std::sync::OnceLock<::lrpar::ParserData<#storaget>> - = ::std::sync::OnceLock::new(); - DATA.get_or_init( - || { - // We have to call reconstitute like this because the config parameter takes a trait - // which uses const generics. Thus the two config parameters here are not actually of the same type. - match __SERIALISATION_FORMAT { - ::lrpar::ctbuilder::SerialisationFormat::FixedSizeInteger => { - ::lrpar::ctbuilder::_reconstitute(__GRM_DATA, __STABLE_DATA, ::lrpar::ctbuilder::wincode::config::Configuration::default().with_fixint_encoding()) - } - ::lrpar::ctbuilder::SerialisationFormat::VariableSizedInteger => { - ::lrpar::ctbuilder::_reconstitute(__GRM_DATA, __STABLE_DATA, ::lrpar::ctbuilder::wincode::config::Configuration::default().with_varint_encoding()) - } - _ => { - panic!("Parser source was generated using unknown `SerialisationFormat`: {:?}", #serialisation_format_str) - } - } - } - ) - } - - #[allow(dead_code)] - pub fn parse #generics ( - lexer: &'lexer dyn ::lrpar::NonStreamingLexer<'input, #lexertypest>, - #parse_fn_parse_param - ) -> #parse_fn_return_ty - #where_clause - { - let __data = __lrpar_parser_data(); - let grm = __data.grm(); - let stable = __data.stable(); - #run_parser - } - }) - } - /// Generate the wrappers that call user actions fn gen_wrappers( &self, @@ -1306,21 +1127,6 @@ where }); Ok(wrappers) } - - /// Return the `RIdx` of the %start rule in the grammar (which will not be the same as - /// grm.start_rule_idx because the latter has an additional rule insert by cfgrammar - /// which then calls the user's %start rule). - fn user_start_ridx( - &self, - code_gen: &ParserCodegen, - ) -> RIdx { - let grm = code_gen.grm(); - debug_assert_eq!(grm.prod(grm.start_prod()).len(), 1); - match grm.prod(grm.start_prod())[0] { - Symbol::Rule(ridx) => ridx, - _ => unreachable!(), - } - } } /// Bundles `YaccGrammar` + `StateTable` so that generated parsers can hold From a521c263d8581cba4beab0d607bcdb6060e80501 Mon Sep 17 00:00:00 2001 From: matt rice Date: Thu, 6 Aug 2026 20:01:20 -0700 Subject: [PATCH 10/14] Move `gen_wrappers` to codegen module --- lrpar/src/lib/codegen.rs | 174 +++++++++++++++++++++++++++++++++++- lrpar/src/lib/ctbuilder.rs | 178 +------------------------------------ 2 files changed, 173 insertions(+), 179 deletions(-) diff --git a/lrpar/src/lib/codegen.rs b/lrpar/src/lib/codegen.rs index 2e07d1aac..eb425d033 100644 --- a/lrpar/src/lib/codegen.rs +++ b/lrpar/src/lib/codegen.rs @@ -12,10 +12,7 @@ use std::{ use crate::{ LexerTypes, RecoveryKind, RustEdition, SerialisationFormat, Visibility, - ctbuilder::{ - ACTION_PREFIX, ACTIONS_KIND, ACTIONS_KIND_PREFIX, CTConflictsError, ERROR, FixIntConfig, - GLOBAL_PREFIX, VarIntConfig, indent, make_generics, - }, + ctbuilder::{CTConflictsError, ERROR, FixIntConfig, VarIntConfig, indent, make_generics}, diagnostics::{DiagnosticFormatter, SpannedDiagnosticFormatter}, }; @@ -31,6 +28,12 @@ use quote::{ToTokens, TokenStreamExt, format_ident, quote}; use syn::Generics; use wincode::SchemaWrite; +const ACTION_PREFIX: &str = "__gt_"; +const GLOBAL_PREFIX: &str = "__GT_"; +const ACTIONS_KIND: &str = "__GtActionsKind"; +const ACTIONS_KIND_PREFIX: &str = "Ak"; +const ACTIONS_KIND_HIDDEN: &str = "__GtActionsKindHidden"; + pub(crate) struct ParserSrcEnv<'a> { src: &'a str, // We store the path here so we can generate a module name from it if needed. @@ -883,6 +886,169 @@ where }) } + /// Generate the wrappers that call user actions + pub(crate) fn gen_wrappers( + &self, + build_env: &ParserBuildEnv, + ) -> Result> { + let grm = self.grm(); + let parsed_parse_generics = make_generics(grm.parse_generics().as_deref())?; + let (generics, type_generics, where_clause) = parsed_parse_generics.split_for_impl(); + + let (parse_paramname, parse_paramdef); + match grm.parse_param() { + Some((name, tyname)) => { + parse_paramname = str::parse::(name)?; + let ty = str::parse::(tyname)?; + parse_paramdef = quote!(#parse_paramname: #ty); + } + None => { + parse_paramname = quote!(()); + parse_paramdef = quote! {_: ()}; + } + }; + + let mut wrappers = TokenStream::new(); + for pidx in grm.iter_pidxs() { + let ridx = grm.prod_to_rule(pidx); + + // Iterate over all $-arguments and replace them with their respective + // element from the argument vector (e.g. $1 is replaced by args[0]). At + // the same time extract &str from tokens and actiontype from nonterminals. + let wrapper_fn = format_ident!("{}wrapper_{}", ACTION_PREFIX, usize::from(pidx)); + let ridx_var = format_ident!("{}ridx", ACTION_PREFIX); + let lexer_var = format_ident!("{}lexer", ACTION_PREFIX); + let span_var = format_ident!("{}span", ACTION_PREFIX); + let args_var = format_ident!("{}args", ACTION_PREFIX); + let storaget = str::parse::(type_name::())?; + let lexertypest = str::parse::(type_name::())?; + let actionskind = str::parse::(ACTIONS_KIND)?; + let edition_lifetime = if build_env.rust_edition() != RustEdition::Rust2015 { + Some(quote!('_,)) + } else { + None + }; + let mut wrapper_fn_body = TokenStream::new(); + if grm.action(pidx).is_some() { + // Unpack the arguments passed to us by the drain + for i in 0..grm.prod(pidx).len() { + let arg = format_ident!("{}arg_{}", ACTION_PREFIX, i + 1); + wrapper_fn_body.extend(match grm.prod(pidx)[i] { + Symbol::Rule(ref_ridx) => { + let ref_ridx = usize::from(ref_ridx); + let actionvariant = format_ident!("{}{}", ACTIONS_KIND_PREFIX, ref_ridx); + quote! { + #[allow(clippy::let_unit_value)] + let #arg = match #args_var.next().unwrap() { + ::lrpar::parser::AStackType::ActionType(#actionskind::#type_generics::#actionvariant(x)) => x, + _ => unreachable!() + }; + } + } + Symbol::Token(_) => { + quote! { + let #arg = match #args_var.next().unwrap() { + ::lrpar::parser::AStackType::Lexeme(l) => { + if l.faulty() { + Err(l) + } else { + Ok(l) + } + }, + ::lrpar::parser::AStackType::ActionType(_) => unreachable!() + }; + } + } + }) + } + + // Call the user code + let args = (0..grm.prod(pidx).len()) + .map(|i| format_ident!("{}arg_{}", ACTION_PREFIX, i + 1)) + .collect::>(); + let action_fn = format_ident!("{}action_{}", ACTION_PREFIX, usize::from(pidx)); + let actionsvariant = format_ident!("{}{}", ACTIONS_KIND_PREFIX, usize::from(ridx)); + + wrapper_fn_body.extend(match grm.actiontype(ridx) { + Some(s) if s == "()" => { + // If the rule `r` that we're calling has the unit type then Clippy will warn that + // `enum::A(wrapper_r())` is pointless. We thus have to split it into two: + // `wrapper_r(); enum::A(())`. + quote! { + #action_fn(#ridx_var, #lexer_var, #span_var, #parse_paramname, #(#args,)*); + #actionskind::#type_generics::#actionsvariant(()) + } + } + _ => { + quote! { + #actionskind::#type_generics::#actionsvariant(#action_fn(#ridx_var, #lexer_var, #span_var, #parse_paramname, #(#args,)*)) + } + } + }) + } else if pidx == grm.start_prod() { + wrapper_fn_body.extend(quote!(unreachable!())); + } else { + unreachable!( + "Production in rule '{}' must have an action body, which should have been handled by gen_user_actions.", + grm.rule_name_str(grm.prod_to_rule(pidx)) + ); + }; + + let attrib = if pidx == grm.start_prod() { + // The start prod has an unreachable body so it doesn't use it's variables. + Some(quote!(#[allow(unused_variables)])) + } else { + None + }; + wrappers.extend(quote! { + #attrib + fn #wrapper_fn #generics ( + #ridx_var: ::cfgrammar::RIdx<#storaget>, + #lexer_var: &'lexer dyn ::lrpar::NonStreamingLexer<'input, #lexertypest>, + #span_var: ::cfgrammar::Span, + mut #args_var: ::std::vec::Drain<#edition_lifetime ::lrpar::parser::AStackType<<#lexertypest as ::lrpar::LexerTypes>::LexemeT, #actionskind #type_generics>>, + #parse_paramdef + ) -> #actionskind #type_generics + #where_clause + { + #wrapper_fn_body + } + }) + } + let mut actionskindvariants = Vec::new(); + let actionskindhidden = format_ident!("_{}", ACTIONS_KIND_HIDDEN); + let actionskind = str::parse::(ACTIONS_KIND).unwrap(); + let mut phantom_data_type = Vec::new(); + for ridx in grm.iter_rules() { + if let Some(actiont) = grm.actiontype(ridx) { + let actionskindvariant = + format_ident!("{}{}", ACTIONS_KIND_PREFIX, usize::from(ridx)); + let actiont = str::parse::(actiont).unwrap(); + actionskindvariants.push(quote! { + #actionskindvariant(#actiont) + }) + } + } + for lifetime in parsed_parse_generics.lifetimes() { + let lifetime = &lifetime.lifetime; + phantom_data_type.push(quote! { &#lifetime () }); + } + for type_param in parsed_parse_generics.type_params() { + let ident = &type_param.ident; + phantom_data_type.push(quote! { #ident }); + } + actionskindvariants.push(quote! { + #actionskindhidden(::std::marker::PhantomData<(#(#phantom_data_type,)*)>) + }); + wrappers.extend(quote! { + #[allow(dead_code)] + enum #actionskind #generics #where_clause { + #(#actionskindvariants,)* + } + }); + Ok(wrappers) + } + /// Return the `RIdx` of the %start rule in the grammar (which will not be the same as /// grm.start_rule_idx because the latter has an additional rule insert by cfgrammar /// which then calls the user's %start rule). diff --git a/lrpar/src/lib/ctbuilder.rs b/lrpar/src/lib/ctbuilder.rs index 15e7e7ac5..4972412a9 100644 --- a/lrpar/src/lib/ctbuilder.rs +++ b/lrpar/src/lib/ctbuilder.rs @@ -1,7 +1,6 @@ //! Build grammars at compile-time so that they can be statically included into a binary. use std::{ - any::type_name, collections::{HashMap, HashSet}, env::{current_dir, var}, error::Error, @@ -24,7 +23,7 @@ use crate::{ use crate::unstable_api::UnstableApi; use cfgrammar::{ - Location, Symbol, + Location, header::{Header, HeaderError, HeaderErrorKind, HeaderValue, Namespaced, Setting, Value}, markmap::{Entry, MergeBehavior}, yacc::{YaccGrammar, YaccKind, YaccOriginalActionKind, ast::ASTWithValidityInfo}, @@ -33,18 +32,11 @@ use filetime::FileTime; use lrtable::{StateGraph, StateTable, statetable::Conflicts}; use num_traits::{AsPrimitive, PrimInt, Unsigned}; use proc_macro2::TokenStream; -use quote::{ToTokens, format_ident, quote}; +use quote::{ToTokens, quote}; use syn::{Generics, parse_quote}; use wincode::{SchemaRead, SchemaReadOwned, SchemaWrite}; -pub(crate) const ACTION_PREFIX: &str = "__gt_"; -pub(crate) const GLOBAL_PREFIX: &str = "__GT_"; -pub(crate) const ACTIONS_KIND: &str = "__GtActionsKind"; -pub(crate) const ACTIONS_KIND_PREFIX: &str = "Ak"; -pub(crate) const ACTIONS_KIND_HIDDEN: &str = "__GtActionsKindHidden"; - const RUST_FILE_EXT: &str = "rs"; - const WARNING: &str = "[Warning]"; pub(crate) const ERROR: &str = "[Error]"; @@ -902,7 +894,7 @@ where let parse_function = code_gen.gen_parse_function(build_env)?; let action_wrappers = match build_env.yacc_kind() { YaccKind::Original(YaccOriginalActionKind::UserAction) | YaccKind::Grmtools => { - Some(self.gen_wrappers(code_gen, build_env)?) + Some(code_gen.gen_wrappers(build_env)?) } YaccKind::Original(YaccOriginalActionKind::NoAction) | YaccKind::Original(YaccOriginalActionKind::GenericParseTree) => None, @@ -963,170 +955,6 @@ where f.write_all(cache.as_bytes())?; Ok(()) } - - /// Generate the wrappers that call user actions - fn gen_wrappers( - &self, - code_gen: &ParserCodegen, - build_env: &ParserBuildEnv, - ) -> Result> { - let grm = code_gen.grm(); - let parsed_parse_generics = make_generics(grm.parse_generics().as_deref())?; - let (generics, type_generics, where_clause) = parsed_parse_generics.split_for_impl(); - - let (parse_paramname, parse_paramdef); - match grm.parse_param() { - Some((name, tyname)) => { - parse_paramname = str::parse::(name)?; - let ty = str::parse::(tyname)?; - parse_paramdef = quote!(#parse_paramname: #ty); - } - None => { - parse_paramname = quote!(()); - parse_paramdef = quote! {_: ()}; - } - }; - - let mut wrappers = TokenStream::new(); - for pidx in grm.iter_pidxs() { - let ridx = grm.prod_to_rule(pidx); - - // Iterate over all $-arguments and replace them with their respective - // element from the argument vector (e.g. $1 is replaced by args[0]). At - // the same time extract &str from tokens and actiontype from nonterminals. - let wrapper_fn = format_ident!("{}wrapper_{}", ACTION_PREFIX, usize::from(pidx)); - let ridx_var = format_ident!("{}ridx", ACTION_PREFIX); - let lexer_var = format_ident!("{}lexer", ACTION_PREFIX); - let span_var = format_ident!("{}span", ACTION_PREFIX); - let args_var = format_ident!("{}args", ACTION_PREFIX); - let storaget = str::parse::(type_name::())?; - let lexertypest = str::parse::(type_name::())?; - let actionskind = str::parse::(ACTIONS_KIND)?; - let edition_lifetime = if build_env.rust_edition() != RustEdition::Rust2015 { - Some(quote!('_,)) - } else { - None - }; - let mut wrapper_fn_body = TokenStream::new(); - if grm.action(pidx).is_some() { - // Unpack the arguments passed to us by the drain - for i in 0..grm.prod(pidx).len() { - let arg = format_ident!("{}arg_{}", ACTION_PREFIX, i + 1); - wrapper_fn_body.extend(match grm.prod(pidx)[i] { - Symbol::Rule(ref_ridx) => { - let ref_ridx = usize::from(ref_ridx); - let actionvariant = format_ident!("{}{}", ACTIONS_KIND_PREFIX, ref_ridx); - quote! { - #[allow(clippy::let_unit_value)] - let #arg = match #args_var.next().unwrap() { - ::lrpar::parser::AStackType::ActionType(#actionskind::#type_generics::#actionvariant(x)) => x, - _ => unreachable!() - }; - } - } - Symbol::Token(_) => { - quote! { - let #arg = match #args_var.next().unwrap() { - ::lrpar::parser::AStackType::Lexeme(l) => { - if l.faulty() { - Err(l) - } else { - Ok(l) - } - }, - ::lrpar::parser::AStackType::ActionType(_) => unreachable!() - }; - } - } - }) - } - - // Call the user code - let args = (0..grm.prod(pidx).len()) - .map(|i| format_ident!("{}arg_{}", ACTION_PREFIX, i + 1)) - .collect::>(); - let action_fn = format_ident!("{}action_{}", ACTION_PREFIX, usize::from(pidx)); - let actionsvariant = format_ident!("{}{}", ACTIONS_KIND_PREFIX, usize::from(ridx)); - - wrapper_fn_body.extend(match grm.actiontype(ridx) { - Some(s) if s == "()" => { - // If the rule `r` that we're calling has the unit type then Clippy will warn that - // `enum::A(wrapper_r())` is pointless. We thus have to split it into two: - // `wrapper_r(); enum::A(())`. - quote! { - #action_fn(#ridx_var, #lexer_var, #span_var, #parse_paramname, #(#args,)*); - #actionskind::#type_generics::#actionsvariant(()) - } - } - _ => { - quote! { - #actionskind::#type_generics::#actionsvariant(#action_fn(#ridx_var, #lexer_var, #span_var, #parse_paramname, #(#args,)*)) - } - } - }) - } else if pidx == grm.start_prod() { - wrapper_fn_body.extend(quote!(unreachable!())); - } else { - unreachable!( - "Production in rule '{}' must have an action body, which should have been handled by gen_user_actions.", - grm.rule_name_str(grm.prod_to_rule(pidx)) - ); - }; - - let attrib = if pidx == grm.start_prod() { - // The start prod has an unreachable body so it doesn't use it's variables. - Some(quote!(#[allow(unused_variables)])) - } else { - None - }; - wrappers.extend(quote! { - #attrib - fn #wrapper_fn #generics ( - #ridx_var: ::cfgrammar::RIdx<#storaget>, - #lexer_var: &'lexer dyn ::lrpar::NonStreamingLexer<'input, #lexertypest>, - #span_var: ::cfgrammar::Span, - mut #args_var: ::std::vec::Drain<#edition_lifetime ::lrpar::parser::AStackType<<#lexertypest as ::lrpar::LexerTypes>::LexemeT, #actionskind #type_generics>>, - #parse_paramdef - ) -> #actionskind #type_generics - #where_clause - { - #wrapper_fn_body - } - }) - } - let mut actionskindvariants = Vec::new(); - let actionskindhidden = format_ident!("_{}", ACTIONS_KIND_HIDDEN); - let actionskind = str::parse::(ACTIONS_KIND).unwrap(); - let mut phantom_data_type = Vec::new(); - for ridx in grm.iter_rules() { - if let Some(actiont) = grm.actiontype(ridx) { - let actionskindvariant = - format_ident!("{}{}", ACTIONS_KIND_PREFIX, usize::from(ridx)); - let actiont = str::parse::(actiont).unwrap(); - actionskindvariants.push(quote! { - #actionskindvariant(#actiont) - }) - } - } - for lifetime in parsed_parse_generics.lifetimes() { - let lifetime = &lifetime.lifetime; - phantom_data_type.push(quote! { &#lifetime () }); - } - for type_param in parsed_parse_generics.type_params() { - let ident = &type_param.ident; - phantom_data_type.push(quote! { #ident }); - } - actionskindvariants.push(quote! { - #actionskindhidden(::std::marker::PhantomData<(#(#phantom_data_type,)*)>) - }); - wrappers.extend(quote! { - #[allow(dead_code)] - enum #actionskind #generics #where_clause { - #(#actionskindvariants,)* - } - }); - Ok(wrappers) - } } /// Bundles `YaccGrammar` + `StateTable` so that generated parsers can hold From 6a39e1b1899965f68c7dbeb5a4c79660fb77cbac Mon Sep 17 00:00:00 2001 From: matt rice Date: Thu, 6 Aug 2026 20:19:53 -0700 Subject: [PATCH 11/14] Move most of `output_file` to the codegen module --- lrpar/src/lib/codegen.rs | 77 ++++++++++++++++++++++++++++++++++++++ lrpar/src/lib/ctbuilder.rs | 74 +----------------------------------- 2 files changed, 79 insertions(+), 72 deletions(-) diff --git a/lrpar/src/lib/codegen.rs b/lrpar/src/lib/codegen.rs index eb425d033..8ba844618 100644 --- a/lrpar/src/lib/codegen.rs +++ b/lrpar/src/lib/codegen.rs @@ -443,6 +443,83 @@ where (self.grm, self.sgraph, self.stable) } + pub(crate) fn generate( + &self, + src_env: &ParserSrcEnv, + build_env: &ParserBuildEnv, + ) -> Result> { + let mod_name = build_env.derived_mod_name(); + let visibility = build_env.visibility(); + let user_actions = if let YaccKind::Original(YaccOriginalActionKind::UserAction) + | YaccKind::Grmtools = build_env.yacc_kind() + { + Some(self.gen_user_actions(src_env)?) + } else { + None + }; + let rule_consts = self.gen_rule_consts()?; + let token_epp = self.gen_token_epp()?; + let parse_function = self.gen_parse_function(build_env)?; + let action_wrappers = match build_env.yacc_kind() { + YaccKind::Original(YaccOriginalActionKind::UserAction) | YaccKind::Grmtools => { + Some(self.gen_wrappers(build_env)?) + } + YaccKind::Original(YaccOriginalActionKind::NoAction) + | YaccKind::Original(YaccOriginalActionKind::GenericParseTree) => None, + _ => unreachable!(), + }; + + let additional_decls = if let YaccKind::Original(YaccOriginalActionKind::GenericParseTree) = + build_env.yacc_kind() + { + // `lrpar::Node`` is deprecated within the lrpar crate, but not from within this module, + // Once it is removed from `lrpar`, we should move the declaration here entirely. + Some(quote! { + #[allow(unused_imports)] + pub use ::lrpar::parser::_deprecated_moved_::Node; + }) + } else { + None + }; + + let mod_name = + match syn::parse_str::(mod_name) { + Ok(s) => s, + Err(e) => return Err(format!( + "CTParserBuilder::mod_name(\"{}\") is not a valid rust identifier due to '{}'", + mod_name, e + ) + .into()), + }; + let out_tokens = quote! { + #visibility mod #mod_name { + // At the top so that `user_actions` may contain #![inner_attribute] + #user_actions + mod _parser_ { + #![allow(clippy::type_complexity)] + #![allow(clippy::unnecessary_wraps)] + #![deny(unsafe_code)] + #[allow(unused_imports)] + use super::*; + #additional_decls + #parse_function + #rule_consts + #token_epp + #action_wrappers + } // End of `mod _parser_` + #[allow(unused_imports)] + pub use _parser_::*; + #[allow(unused_imports)] + use ::lrpar::Lexeme; + } // End of `mod #mod_name` + }; + // Try and run a code formatter on the generated code. + let unformatted = out_tokens.to_string(); + Ok(syn::parse_str(&unformatted) + .map(|syntax_tree| prettyplease::unparse(&syntax_tree)) + .unwrap_or(unformatted)) + } + /// Generate the cache, which determines if anything's changed enough that we need to /// regenerate outputs and force rustc to recompile. fn gen_cache( diff --git a/lrpar/src/lib/ctbuilder.rs b/lrpar/src/lib/ctbuilder.rs index 4972412a9..1309f7211 100644 --- a/lrpar/src/lib/ctbuilder.rs +++ b/lrpar/src/lib/ctbuilder.rs @@ -26,7 +26,7 @@ use cfgrammar::{ Location, header::{Header, HeaderError, HeaderErrorKind, HeaderValue, Namespaced, Setting, Value}, markmap::{Entry, MergeBehavior}, - yacc::{YaccGrammar, YaccKind, YaccOriginalActionKind, ast::ASTWithValidityInfo}, + yacc::{YaccGrammar, YaccKind, ast::ASTWithValidityInfo}, }; use filetime::FileTime; use lrtable::{StateGraph, StateTable, statetable::Conflicts}; @@ -879,77 +879,7 @@ where src_env: &ParserSrcEnv, build_env: &ParserBuildEnv<'_, LexerTypesT>, ) -> Result<(), Box> { - let mod_name = build_env.derived_mod_name(); - let visibility = build_env.visibility(); - let user_actions = if let YaccKind::Original(YaccOriginalActionKind::UserAction) - | YaccKind::Grmtools = build_env.yacc_kind() - { - Some(code_gen.gen_user_actions(src_env)?) - } else { - None - }; - - let rule_consts = code_gen.gen_rule_consts()?; - let token_epp = code_gen.gen_token_epp()?; - let parse_function = code_gen.gen_parse_function(build_env)?; - let action_wrappers = match build_env.yacc_kind() { - YaccKind::Original(YaccOriginalActionKind::UserAction) | YaccKind::Grmtools => { - Some(code_gen.gen_wrappers(build_env)?) - } - YaccKind::Original(YaccOriginalActionKind::NoAction) - | YaccKind::Original(YaccOriginalActionKind::GenericParseTree) => None, - _ => unreachable!(), - }; - - let additional_decls = if let YaccKind::Original(YaccOriginalActionKind::GenericParseTree) = - build_env.yacc_kind() - { - // `lrpar::Node`` is deprecated within the lrpar crate, but not from within this module, - // Once it is removed from `lrpar`, we should move the declaration here entirely. - Some(quote! { - #[allow(unused_imports)] - pub use ::lrpar::parser::_deprecated_moved_::Node; - }) - } else { - None - }; - - let mod_name = - match syn::parse_str::(mod_name) { - Ok(s) => s, - Err(e) => return Err(format!( - "CTParserBuilder::mod_name(\"{}\") is not a valid rust identifier due to '{}'", - mod_name, e - ) - .into()), - }; - let out_tokens = quote! { - #visibility mod #mod_name { - // At the top so that `user_actions` may contain #![inner_attribute] - #user_actions - mod _parser_ { - #![allow(clippy::type_complexity)] - #![allow(clippy::unnecessary_wraps)] - #![deny(unsafe_code)] - #[allow(unused_imports)] - use super::*; - #additional_decls - #parse_function - #rule_consts - #token_epp - #action_wrappers - } // End of `mod _parser_` - #[allow(unused_imports)] - pub use _parser_::*; - #[allow(unused_imports)] - use ::lrpar::Lexeme; - } // End of `mod #mod_name` - }; - // Try and run a code formatter on the generated code. - let unformatted = out_tokens.to_string(); - let outs = syn::parse_str(&unformatted) - .map(|syntax_tree| prettyplease::unparse(&syntax_tree)) - .unwrap_or(unformatted); + let outs = code_gen.generate(src_env, build_env)?; let mut f = File::create(outp_rs)?; f.write_all(outs.as_bytes())?; f.write_all(cache.as_bytes())?; From b55acefd18494211f5ab031237bed636ad7dce58 Mon Sep 17 00:00:00 2001 From: matt rice Date: Thu, 6 Aug 2026 20:33:02 -0700 Subject: [PATCH 12/14] Move additional codegen helpers --- lrpar/src/lib/codegen.rs | 29 +++++++++++++++++++++++++++-- lrpar/src/lib/ctbuilder.rs | 27 --------------------------- 2 files changed, 27 insertions(+), 29 deletions(-) diff --git a/lrpar/src/lib/codegen.rs b/lrpar/src/lib/codegen.rs index 8ba844618..2284b29c6 100644 --- a/lrpar/src/lib/codegen.rs +++ b/lrpar/src/lib/codegen.rs @@ -12,7 +12,7 @@ use std::{ use crate::{ LexerTypes, RecoveryKind, RustEdition, SerialisationFormat, Visibility, - ctbuilder::{CTConflictsError, ERROR, FixIntConfig, VarIntConfig, indent, make_generics}, + ctbuilder::{CTConflictsError, ERROR, FixIntConfig, VarIntConfig, indent}, diagnostics::{DiagnosticFormatter, SpannedDiagnosticFormatter}, }; @@ -25,7 +25,7 @@ use cfgrammar::{ use lrtable::{Minimiser, StateGraph, StateTable, from_yacc}; use proc_macro2::{Literal, TokenStream}; use quote::{ToTokens, TokenStreamExt, format_ident, quote}; -use syn::Generics; +use syn::{Generics, parse_quote}; use wincode::SchemaWrite; const ACTION_PREFIX: &str = "__gt_"; @@ -1245,3 +1245,28 @@ impl Visibility { } } } + +impl ToTokens for SerialisationFormat { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.extend(match self { + SerialisationFormat::FixedSizeInteger => { + quote! {::lrpar::ctbuilder::SerialisationFormat::FixedSizeInteger} + } + SerialisationFormat::VariableSizedInteger => { + quote! {::lrpar::ctbuilder::SerialisationFormat::VariableSizedInteger} + } + }) + } +} + +pub(crate) fn make_generics(parse_generics: Option<&str>) -> Result> { + if let Some(parse_generics) = parse_generics { + let tokens = str::parse::(parse_generics)?; + match syn::parse2(quote!(<'lexer, 'input: 'lexer, #tokens>)) { + Ok(res) => Ok(res), + Err(err) => Err(format!("unable to parse %parse-generics: {}", err).into()), + } + } else { + Ok(parse_quote!(<'lexer, 'input: 'lexer>)) + } +} \ No newline at end of file diff --git a/lrpar/src/lib/ctbuilder.rs b/lrpar/src/lib/ctbuilder.rs index 1309f7211..310f7929b 100644 --- a/lrpar/src/lib/ctbuilder.rs +++ b/lrpar/src/lib/ctbuilder.rs @@ -31,9 +31,6 @@ use cfgrammar::{ use filetime::FileTime; use lrtable::{StateGraph, StateTable, statetable::Conflicts}; use num_traits::{AsPrimitive, PrimInt, Unsigned}; -use proc_macro2::TokenStream; -use quote::{ToTokens, quote}; -use syn::{Generics, parse_quote}; use wincode::{SchemaRead, SchemaReadOwned, SchemaWrite}; const RUST_FILE_EXT: &str = "rs"; @@ -213,18 +210,6 @@ impl TryFrom<&Value> for SerialisationFormat { // We export this for generated code to refer to. #[doc(hidden)] pub use wincode; -impl ToTokens for SerialisationFormat { - fn to_tokens(&self, tokens: &mut TokenStream) { - tokens.extend(match self { - SerialisationFormat::FixedSizeInteger => { - quote! {::lrpar::ctbuilder::SerialisationFormat::FixedSizeInteger} - } - SerialisationFormat::VariableSizedInteger => { - quote! {::lrpar::ctbuilder::SerialisationFormat::VariableSizedInteger} - } - }) - } -} /// A `CTParserBuilder` allows one to specify the criteria for building a statically generated /// parser. @@ -1004,18 +989,6 @@ pub(crate) fn indent(indent: &str, s: &str) -> String { format!("{indent}{}\n", s.trim_end_matches('\n')).replace('\n', &format!("\n{}", indent)) } -pub(crate) fn make_generics(parse_generics: Option<&str>) -> Result> { - if let Some(parse_generics) = parse_generics { - let tokens = str::parse::(parse_generics)?; - match syn::parse2(quote!(<'lexer, 'input: 'lexer, #tokens>)) { - Ok(res) => Ok(res), - Err(err) => Err(format!("unable to parse %parse-generics: {}", err).into()), - } - } else { - Ok(parse_quote!(<'lexer, 'input: 'lexer>)) - } -} - // Tests dealing with the filesystem not supported under wasm32 #[cfg(all(not(target_arch = "wasm32"), test))] mod test { From 22b6d7f1822a09b9dfdc98c558f2a95cebfdb8e2 Mon Sep 17 00:00:00 2001 From: matt rice Date: Thu, 6 Aug 2026 20:35:15 -0700 Subject: [PATCH 13/14] Remove extraneous `pub(crate)` since everything is moved --- lrpar/src/lib/codegen.rs | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/lrpar/src/lib/codegen.rs b/lrpar/src/lib/codegen.rs index 2284b29c6..31ecc4eca 100644 --- a/lrpar/src/lib/codegen.rs +++ b/lrpar/src/lib/codegen.rs @@ -577,10 +577,7 @@ where } /// Generate the user action functions (if any). - pub(crate) fn gen_user_actions( - &self, - src_env: &ParserSrcEnv, - ) -> Result> { + fn gen_user_actions(&self, src_env: &ParserSrcEnv) -> Result> { let grm = self.grm(); let diag = src_env.yacc_diag(); let programs = grm @@ -746,7 +743,7 @@ where }) } - pub(crate) fn gen_rule_consts(&self) -> Result { + fn gen_rule_consts(&self) -> Result { let grm = self.grm(); let mut toks = TokenStream::new(); for ridx in grm.iter_rules() { @@ -763,7 +760,7 @@ where Ok(toks) } - pub(crate) fn gen_token_epp(&self) -> Result { + fn gen_token_epp(&self) -> Result { let grm = self.grm(); let mut tidxs = Vec::new(); for tidx in grm.iter_tidxs() { @@ -786,7 +783,7 @@ where } /// Generate the main parse() function for the output file. - pub(crate) fn gen_parse_function( + fn gen_parse_function( &self, build_env: &ParserBuildEnv, ) -> Result> { @@ -964,7 +961,7 @@ where } /// Generate the wrappers that call user actions - pub(crate) fn gen_wrappers( + fn gen_wrappers( &self, build_env: &ParserBuildEnv, ) -> Result> { @@ -1269,4 +1266,4 @@ pub(crate) fn make_generics(parse_generics: Option<&str>) -> Result)) } -} \ No newline at end of file +} From cf24f1da4981054cc8abb49d6bcafe5bf09aa415 Mon Sep 17 00:00:00 2001 From: matt rice Date: Thu, 6 Aug 2026 20:39:29 -0700 Subject: [PATCH 14/14] Remove/fix any lint suppressions --- lrpar/src/lib/codegen.rs | 13 +------------ lrpar/src/lib/ctbuilder.rs | 4 +++- 2 files changed, 4 insertions(+), 13 deletions(-) diff --git a/lrpar/src/lib/codegen.rs b/lrpar/src/lib/codegen.rs index 31ecc4eca..503db49e3 100644 --- a/lrpar/src/lib/codegen.rs +++ b/lrpar/src/lib/codegen.rs @@ -1,6 +1,3 @@ -#![deny(unfulfilled_lint_expectations)] -#![expect(dead_code)] - use std::{ any::type_name, error::Error, @@ -152,10 +149,6 @@ impl<'a> ParserSrcEnv<'a> { &mut self.header } - pub(crate) fn header(&self) -> &Header { - &self.header - } - fn merge_headers(&mut self) -> Result<(), Box> { let (parsed_header, _) = self.parse_header()?; Ok(self.header.merge_from(parsed_header)?) @@ -429,10 +422,6 @@ where &self.stable } - pub(crate) fn sgraph(&self) -> &StateGraph { - &self.sgraph - } - pub(crate) fn take_parser( self, ) -> ( @@ -528,7 +517,7 @@ where build_env: &ParserBuildEnv, ) -> TokenStream { let grm = self.grm(); - let build_time = env!("VERGEN_BUILD_TIMESTAMP"); + let build_time = &self.timestamp; let grammar_path = src_env.path().to_string_lossy(); let mod_name = QuoteOption(build_env.specified_mod_name()); let visibility = build_env.visibility().to_variant_tokens(); diff --git a/lrpar/src/lib/ctbuilder.rs b/lrpar/src/lib/ctbuilder.rs index 310f7929b..761e2edd8 100644 --- a/lrpar/src/lib/ctbuilder.rs +++ b/lrpar/src/lib/ctbuilder.rs @@ -618,7 +618,9 @@ where .mod_name(self.mod_name) .show_warnings(self.show_warnings) .error_on_conflicts(self.error_on_conflicts) - .warnings_are_errors(self.warnings_are_errors); + .warnings_are_errors(self.warnings_are_errors) + .visibility(self.visibility.clone()) + .rust_edition(self.rust_edition); let build_env = src_env.build_env::(build_args)?; // Temporarily we update self.yacckind and self.recoverer from the build_env // Until codegen reads these variables from the build_env directly.