Skip to content

Commit

Permalink
Merge pull request #254 from bgw/grammar-type-params
Browse files Browse the repository at this point in the history 
Accept type parameters on grammar (e.g. lifetimes) (Fixes #220)
  • Loading branch information
@kevinmehall
kevinmehall committed Mar 14, 2021
2 parents 25e19d2 + 6b19369 commit 2dacb48
Show file tree
Hide file tree
Showing 5 changed files with 68 additions and 35 deletions.
1 change: 1 addition & 0 deletions peg-macros/ast.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ pub struct Grammar {
pub doc: Option<TokenStream>,
pub visibility: Option<TokenStream>,
pub name: Ident,
pub ty_params: Option<Vec<TokenStream>>,
pub args: Vec<(Ident, TokenStream)>,
pub items: Vec<Item>,
pub input_type: TokenStream,
Expand Down
42 changes: 20 additions & 22 deletions peg-macros/grammar.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -84,33 +84,31 @@ pub mod peg {
__parse_IDENT(__input, __state, __err_state, __pos);
match __seq_res {
::peg::RuleResult::Matched(__pos, name) => {
let __seq_res = __parse_grammar_args(
let __seq_res = match __parse_rust_ty_params(
__input,
__state,
__err_state,
__pos,
);
) {
::peg::RuleResult::Matched(__newpos, __value) => {
::peg::RuleResult::Matched(
__newpos,
Some(__value),
)
}
::peg::RuleResult::Failed => {
::peg::RuleResult::Matched(__pos, None)
}
};
match __seq_res {
::peg::RuleResult::Matched(__pos, args) => {
match ::peg::ParseLiteral::parse_string_literal(
__input, __pos, "for",
) {
::peg::RuleResult::Matched(
__pos,
__val,
) => {
let __seq_res = {
let str_start = __pos;
match match __parse_rust_type (__input , __state , __err_state , __pos) { :: peg :: RuleResult :: Matched (pos , _) => :: peg :: RuleResult :: Matched (pos , ()) , :: peg :: RuleResult :: Failed => :: peg :: RuleResult :: Failed , } { :: peg :: RuleResult :: Matched (__newpos , _) => { :: peg :: RuleResult :: Matched (__newpos , :: peg :: ParseSlice :: parse_slice (__input , str_start , __newpos)) } , :: peg :: RuleResult :: Failed => :: peg :: RuleResult :: Failed , }
};
match __seq_res { :: peg :: RuleResult :: Matched (__pos , input_type) => { match :: peg :: ParseLiteral :: parse_string_literal (__input , __pos , "{") { :: peg :: RuleResult :: Matched (__pos , __val) => { { let __seq_res = { let mut __repeat_pos = __pos ; let mut __repeat_value = vec ! () ; loop { let __pos = __repeat_pos ; let __step_res = __parse_item (__input , __state , __err_state , __pos) ; match __step_res { :: peg :: RuleResult :: Matched (__newpos , __value) => { __repeat_pos = __newpos ; __repeat_value . push (__value) ; } , :: peg :: RuleResult :: Failed => { break ; } } } :: peg :: RuleResult :: Matched (__repeat_pos , __repeat_value) } ; match __seq_res { :: peg :: RuleResult :: Matched (__pos , items) => { match :: peg :: ParseLiteral :: parse_string_literal (__input , __pos , "}") { :: peg :: RuleResult :: Matched (__pos , __val) => { :: peg :: RuleResult :: Matched (__pos , (|| { Grammar { doc , visibility , name , args , input_type , items } }) ()) } :: peg :: RuleResult :: Failed => { __err_state . mark_failure (__pos , "\"}\"") ; :: peg :: RuleResult :: Failed } } } :: peg :: RuleResult :: Failed => :: peg :: RuleResult :: Failed , } } } :: peg :: RuleResult :: Failed => { __err_state . mark_failure (__pos , "\"{\"") ; :: peg :: RuleResult :: Failed } } } :: peg :: RuleResult :: Failed => :: peg :: RuleResult :: Failed , }
}
::peg::RuleResult::Failed => {
__err_state
.mark_failure(__pos, "\"for\"");
::peg::RuleResult::Failed
}
}
::peg::RuleResult::Matched(__pos, ty_params) => {
let __seq_res = __parse_grammar_args(
__input,
__state,
__err_state,
__pos,
);
match __seq_res { :: peg :: RuleResult :: Matched (__pos , args) => { match :: peg :: ParseLiteral :: parse_string_literal (__input , __pos , "for") { :: peg :: RuleResult :: Matched (__pos , __val) => { { let __seq_res = { let str_start = __pos ; match match __parse_rust_type (__input , __state , __err_state , __pos) { :: peg :: RuleResult :: Matched (pos , _) => :: peg :: RuleResult :: Matched (pos , ()) , :: peg :: RuleResult :: Failed => :: peg :: RuleResult :: Failed , } { :: peg :: RuleResult :: Matched (__newpos , _) => { :: peg :: RuleResult :: Matched (__newpos , :: peg :: ParseSlice :: parse_slice (__input , str_start , __newpos)) } , :: peg :: RuleResult :: Failed => :: peg :: RuleResult :: Failed , } } ; match __seq_res { :: peg :: RuleResult :: Matched (__pos , input_type) => { match :: peg :: ParseLiteral :: parse_string_literal (__input , __pos , "{") { :: peg :: RuleResult :: Matched (__pos , __val) => { { let __seq_res = { let mut __repeat_pos = __pos ; let mut __repeat_value = vec ! () ; loop { let __pos = __repeat_pos ; let __step_res = __parse_item (__input , __state , __err_state , __pos) ; match __step_res { :: peg :: RuleResult :: Matched (__newpos , __value) => { __repeat_pos = __newpos ; __repeat_value . push (__value) ; } , :: peg :: RuleResult :: Failed => { break ; } } } :: peg :: RuleResult :: Matched (__repeat_pos , __repeat_value) } ; match __seq_res { :: peg :: RuleResult :: Matched (__pos , items) => { match :: peg :: ParseLiteral :: parse_string_literal (__input , __pos , "}") { :: peg :: RuleResult :: Matched (__pos , __val) => { :: peg :: RuleResult :: Matched (__pos , (|| { Grammar { doc , visibility , name , ty_params , args , input_type , items } }) ()) } :: peg :: RuleResult :: Failed => { __err_state . mark_failure (__pos , "\"}\"") ; :: peg :: RuleResult :: Failed } } } :: peg :: RuleResult :: Failed => :: peg :: RuleResult :: Failed , } } } :: peg :: RuleResult :: Failed => { __err_state . mark_failure (__pos , "\"{\"") ; :: peg :: RuleResult :: Failed } } } :: peg :: RuleResult :: Failed => :: peg :: RuleResult :: Failed , } } } :: peg :: RuleResult :: Failed => { __err_state . mark_failure (__pos , "\"for\"") ; :: peg :: RuleResult :: Failed } } } :: peg :: RuleResult :: Failed => :: peg :: RuleResult :: Failed , }
}
::peg::RuleResult::Failed => {
::peg::RuleResult::Failed
Expand Down
4 changes: 2 additions & 2 deletions peg-macros/grammar.rustpeg
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,8 @@ use crate::tokens::FlatTokenStream;
use proc_macro2::{ TokenStream, Ident, Group, Literal, Delimiter, Span };

pub rule peg_grammar() -> Grammar
= doc:rust_doc_comment() visibility:rust_visibility() "grammar" name:IDENT() args:grammar_args() "for" input_type:$(rust_type()) "{" items:item()* "}"
{ Grammar { doc, visibility, name, args, input_type, items } }
= doc:rust_doc_comment() visibility:rust_visibility() "grammar" name:IDENT() ty_params:rust_ty_params()? args:grammar_args() "for" input_type:$(rust_type()) "{" items:item()* "}"
{ Grammar { doc, visibility, name, ty_params, args, input_type, items } }

rule grammar_args() -> Vec<(Ident, TokenStream)>
= "(" args:((i:IDENT() ":" t:$(rust_type()) { (i, t) })**",") ","? ")" { args }
Expand Down
32 changes: 21 additions & 11 deletions peg-macros/translate.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@ fn extra_args_call(grammar: &Grammar) -> TokenStream {
struct Context<'a> {
rules: &'a HashMap<String, &'a Rule>,
rules_from_args: HashSet<String>,
extra_ty_params: &'a Option<Vec<TokenStream>>,
extra_args_call: TokenStream,
extra_args_def: TokenStream,
}
Expand All @@ -62,6 +63,7 @@ pub(crate) fn compile_grammar(grammar: &Grammar) -> TokenStream {
let context = &Context {
rules: &analysis.rules,
rules_from_args: HashSet::new(),
extra_ty_params: &grammar.ty_params,
extra_args_call: extra_args_call(grammar),
extra_args_def: extra_args_def(grammar),
};
Expand Down Expand Up @@ -107,6 +109,7 @@ pub(crate) fn compile_grammar(grammar: &Grammar) -> TokenStream {

let doc = &grammar.doc;
let input_type = &grammar.input_type;
let input_ty_params = ty_params_slice(&grammar.ty_params);
let visibility = &grammar.visibility;

let mut errors = Vec::new();
Expand All @@ -124,8 +127,8 @@ pub(crate) fn compile_grammar(grammar: &Grammar) -> TokenStream {
#visibility mod #name {
#[allow(unused_imports)]
use super::*;
type Input = #input_type;
type PositionRepr = <Input as ::peg::Parse>::PositionRepr;
type Input<#(#input_ty_params),*> = #input_type;
type PositionRepr<#(#input_ty_params),*> = <Input<#(#input_ty_params),*> as ::peg::Parse>::PositionRepr;

#(#errors)*
#(#items)*
Expand Down Expand Up @@ -165,14 +168,19 @@ fn make_parse_state(grammar: &Grammar) -> TokenStream {
}
}

fn rule_params_list(rule: &Rule) -> Vec<TokenStream> {
fn ty_params_slice(ty_params: &Option<Vec<TokenStream>>) -> &[TokenStream] {
ty_params.as_ref().map(|x| &x[..]).unwrap_or(&[])
}

fn rule_params_list(context: &Context, rule: &Rule) -> Vec<TokenStream> {
let extra_ty_params = ty_params_slice(context.extra_ty_params);
let span = rule.span.resolved_at(Span::mixed_site());
rule.params.iter().map(|param| {
let name = &param.name;
match &param.ty {
RuleParamTy::Rust(ty) => quote_spanned!{ span => #name: #ty },
RuleParamTy::Rule(ty) => quote_spanned!{ span =>
#name: impl Fn(&'input Input, &mut ParseState<'input>, &mut ::peg::error::ErrorState, usize) -> ::peg::RuleResult<#ty>
#name: impl Fn(&'input Input<#(#extra_ty_params),*>, &mut ParseState<'input>, &mut ::peg::error::ErrorState, usize) -> ::peg::RuleResult<#ty>
},
}
}).collect()
Expand All @@ -184,7 +192,8 @@ fn compile_rule(context: &Context, rule: &Rule) -> TokenStream {
let name = format_ident!("__parse_{}", rule.name, span=span);
let ret_ty = rule.ret_type.clone().unwrap_or_else(|| quote!(()));
let result_used = rule.ret_type.is_some();
let ty_params = rule.ty_params.as_ref().map(|x| &x[..]).unwrap_or(&[]);
let ty_params = ty_params_slice(&rule.ty_params);
let extra_ty_params = ty_params_slice(context.extra_ty_params);

let mut context = context.clone();
context
Expand Down Expand Up @@ -217,7 +226,7 @@ fn compile_rule(context: &Context, rule: &Rule) -> TokenStream {

let extra_args_def = &context.extra_args_def;

let rule_params = rule_params_list(rule);
let rule_params = rule_params_list(&context, rule);

if rule.cached {
let cache_field = format_ident!("{}_cache", rule.name);
Expand All @@ -236,7 +245,7 @@ fn compile_rule(context: &Context, rule: &Rule) -> TokenStream {
};

quote_spanned! { span =>
fn #name<'input #(, #ty_params)*>(__input: &'input Input, __state: &mut ParseState<'input>, __err_state: &mut ::peg::error::ErrorState, __pos: usize #extra_args_def #(, #rule_params)*) -> ::peg::RuleResult<#ret_ty> {
fn #name<'input #(, #extra_ty_params)* #(, #ty_params)*>(__input: &'input Input<#(#extra_ty_params),*>, __state: &mut ParseState<'input>, __err_state: &mut ::peg::error::ErrorState, __pos: usize #extra_args_def #(, #rule_params)*) -> ::peg::RuleResult<#ret_ty> {
#![allow(non_snake_case, unused)]
if let Some(entry) = __state.#cache_field.get(&__pos) {
#cache_trace
Expand All @@ -249,7 +258,7 @@ fn compile_rule(context: &Context, rule: &Rule) -> TokenStream {
}
} else {
quote_spanned! { span =>
fn #name<'input #(, #ty_params)*>(__input: &'input Input, __state: &mut ParseState<'input>, __err_state: &mut ::peg::error::ErrorState, __pos: usize #extra_args_def #(, #rule_params)*) -> ::peg::RuleResult<#ret_ty> {
fn #name<'input #(, #extra_ty_params)* #(, #ty_params)*>(__input: &'input Input<#(#extra_ty_params),*>, __state: &mut ParseState<'input>, __err_state: &mut ::peg::error::ErrorState, __pos: usize #extra_args_def #(, #rule_params)*) -> ::peg::RuleResult<#ret_ty> {
#![allow(non_snake_case, unused)]
#wrapped_body
}
Expand All @@ -264,8 +273,9 @@ fn compile_rule_export(context: &Context, rule: &Rule) -> TokenStream {
let ret_ty = rule.ret_type.clone().unwrap_or_else(|| quote!(()));
let visibility = &rule.visibility;
let parse_fn = format_ident!("__parse_{}", rule.name.to_string(), span = name.span());
let ty_params = rule.ty_params.as_ref().map(|x| &x[..]).unwrap_or(&[]);
let rule_params = rule_params_list(rule);
let ty_params = ty_params_slice(&rule.ty_params);
let extra_ty_params = ty_params_slice(context.extra_ty_params);
let rule_params = rule_params_list(context, rule);
let rule_params_call: Vec<TokenStream> = rule
.params
.iter()
Expand All @@ -280,7 +290,7 @@ fn compile_rule_export(context: &Context, rule: &Rule) -> TokenStream {

quote_spanned! { span =>
#doc
#visibility fn #name<'input #(, #ty_params)*>(__input: &'input Input #extra_args_def #(, #rule_params)*) -> ::std::result::Result<#ret_ty, ::peg::error::ParseError<PositionRepr>> {
#visibility fn #name<'input #(, #extra_ty_params)* #(, #ty_params)*>(__input: &'input Input<#(#extra_ty_params),*> #extra_args_def #(, #rule_params)*) -> ::std::result::Result<#ret_ty, ::peg::error::ParseError<PositionRepr<#(#extra_ty_params),*>>> {
#![allow(non_snake_case, unused)]

let mut __err_state = ::peg::error::ErrorState::new(::peg::Parse::start(__input));
Expand Down
24 changes: 24 additions & 0 deletions tests/run-pass/lifetimes.rs
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
#[derive(Clone)]
pub struct Token<'text>(&'text str);

peg::parser!{
grammar tokenparser<'t>() for [Token<'t>] {
pub rule program() -> Vec<&'t str> = list()

// add this indirection to ensure that rule args work with a global lifetime
rule commasep<T>(x: rule<T>) -> Vec<T> = v:(x() ** [Token(",")]) [Token(",")]? { v }

rule list() -> Vec<&'t str> = [Token("(")] l:commasep(<string()>) [Token(")")] { l }
rule string() -> &'t str = [Token(inner)] { inner }
}
}

fn main() {
let input = "(one,two)";
assert_eq!(
tokenparser::program(
&[Token(&input[0..1]), Token(&input[1..4]), Token(&input[4..5]), Token(&input[5..8]), Token(&input[8..9])],
),
Ok(vec!["one", "two"])
);
}

0 comments on commit 2dacb48

Please sign in to comment.