Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Accept type parameters on grammar (e.g. lifetimes) (Fixes #220) #254

Merged
merged 1 commit into from
Mar 14, 2021
Merged

Accept type parameters on grammar (e.g. lifetimes) (Fixes #220) #254

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions peg-macros/ast.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ pub struct Grammar {
pub doc: Option<TokenStream>,
pub visibility: Option<TokenStream>,
pub name: Ident,
pub ty_params: Option<Vec<TokenStream>>,
pub args: Vec<(Ident, TokenStream)>,
pub items: Vec<Item>,
pub input_type: TokenStream,
Expand Down
42 changes: 20 additions & 22 deletions peg-macros/grammar.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -84,33 +84,31 @@ pub mod peg {
__parse_IDENT(__input, __state, __err_state, __pos);
match __seq_res {
::peg::RuleResult::Matched(__pos, name) => {
let __seq_res = __parse_grammar_args(
let __seq_res = match __parse_rust_ty_params(
__input,
__state,
__err_state,
__pos,
);
) {
::peg::RuleResult::Matched(__newpos, __value) => {
::peg::RuleResult::Matched(
__newpos,
Some(__value),
)
}
::peg::RuleResult::Failed => {
::peg::RuleResult::Matched(__pos, None)
}
};
match __seq_res {
::peg::RuleResult::Matched(__pos, args) => {
match ::peg::ParseLiteral::parse_string_literal(
__input, __pos, "for",
) {
::peg::RuleResult::Matched(
__pos,
__val,
) => {
let __seq_res = {
let str_start = __pos;
match match __parse_rust_type (__input , __state , __err_state , __pos) { :: peg :: RuleResult :: Matched (pos , _) => :: peg :: RuleResult :: Matched (pos , ()) , :: peg :: RuleResult :: Failed => :: peg :: RuleResult :: Failed , } { :: peg :: RuleResult :: Matched (__newpos , _) => { :: peg :: RuleResult :: Matched (__newpos , :: peg :: ParseSlice :: parse_slice (__input , str_start , __newpos)) } , :: peg :: RuleResult :: Failed => :: peg :: RuleResult :: Failed , }
};
match __seq_res { :: peg :: RuleResult :: Matched (__pos , input_type) => { match :: peg :: ParseLiteral :: parse_string_literal (__input , __pos , "{") { :: peg :: RuleResult :: Matched (__pos , __val) => { { let __seq_res = { let mut __repeat_pos = __pos ; let mut __repeat_value = vec ! () ; loop { let __pos = __repeat_pos ; let __step_res = __parse_item (__input , __state , __err_state , __pos) ; match __step_res { :: peg :: RuleResult :: Matched (__newpos , __value) => { __repeat_pos = __newpos ; __repeat_value . push (__value) ; } , :: peg :: RuleResult :: Failed => { break ; } } } :: peg :: RuleResult :: Matched (__repeat_pos , __repeat_value) } ; match __seq_res { :: peg :: RuleResult :: Matched (__pos , items) => { match :: peg :: ParseLiteral :: parse_string_literal (__input , __pos , "}") { :: peg :: RuleResult :: Matched (__pos , __val) => { :: peg :: RuleResult :: Matched (__pos , (|| { Grammar { doc , visibility , name , args , input_type , items } }) ()) } :: peg :: RuleResult :: Failed => { __err_state . mark_failure (__pos , "\"}\"") ; :: peg :: RuleResult :: Failed } } } :: peg :: RuleResult :: Failed => :: peg :: RuleResult :: Failed , } } } :: peg :: RuleResult :: Failed => { __err_state . mark_failure (__pos , "\"{\"") ; :: peg :: RuleResult :: Failed } } } :: peg :: RuleResult :: Failed => :: peg :: RuleResult :: Failed , }
}
::peg::RuleResult::Failed => {
__err_state
.mark_failure(__pos, "\"for\"");
::peg::RuleResult::Failed
}
}
::peg::RuleResult::Matched(__pos, ty_params) => {
let __seq_res = __parse_grammar_args(
__input,
__state,
__err_state,
__pos,
);
match __seq_res { :: peg :: RuleResult :: Matched (__pos , args) => { match :: peg :: ParseLiteral :: parse_string_literal (__input , __pos , "for") { :: peg :: RuleResult :: Matched (__pos , __val) => { { let __seq_res = { let str_start = __pos ; match match __parse_rust_type (__input , __state , __err_state , __pos) { :: peg :: RuleResult :: Matched (pos , _) => :: peg :: RuleResult :: Matched (pos , ()) , :: peg :: RuleResult :: Failed => :: peg :: RuleResult :: Failed , } { :: peg :: RuleResult :: Matched (__newpos , _) => { :: peg :: RuleResult :: Matched (__newpos , :: peg :: ParseSlice :: parse_slice (__input , str_start , __newpos)) } , :: peg :: RuleResult :: Failed => :: peg :: RuleResult :: Failed , } } ; match __seq_res { :: peg :: RuleResult :: Matched (__pos , input_type) => { match :: peg :: ParseLiteral :: parse_string_literal (__input , __pos , "{") { :: peg :: RuleResult :: Matched (__pos , __val) => { { let __seq_res = { let mut __repeat_pos = __pos ; let mut __repeat_value = vec ! () ; loop { let __pos = __repeat_pos ; let __step_res = __parse_item (__input , __state , __err_state , __pos) ; match __step_res { :: peg :: RuleResult :: Matched (__newpos , __value) => { __repeat_pos = __newpos ; __repeat_value . push (__value) ; } , :: peg :: RuleResult :: Failed => { break ; } } } :: peg :: RuleResult :: Matched (__repeat_pos , __repeat_value) } ; match __seq_res { :: peg :: RuleResult :: Matched (__pos , items) => { match :: peg :: ParseLiteral :: parse_string_literal (__input , __pos , "}") { :: peg :: RuleResult :: Matched (__pos , __val) => { :: peg :: RuleResult :: Matched (__pos , (|| { Grammar { doc , visibility , name , ty_params , args , input_type , items } }) ()) } :: peg :: RuleResult :: Failed => { __err_state . mark_failure (__pos , "\"}\"") ; :: peg :: RuleResult :: Failed } } } :: peg :: RuleResult :: Failed => :: peg :: RuleResult :: Failed , } } } :: peg :: RuleResult :: Failed => { __err_state . mark_failure (__pos , "\"{\"") ; :: peg :: RuleResult :: Failed } } } :: peg :: RuleResult :: Failed => :: peg :: RuleResult :: Failed , } } } :: peg :: RuleResult :: Failed => { __err_state . mark_failure (__pos , "\"for\"") ; :: peg :: RuleResult :: Failed } } } :: peg :: RuleResult :: Failed => :: peg :: RuleResult :: Failed , }
}
::peg::RuleResult::Failed => {
::peg::RuleResult::Failed
Expand Down
4 changes: 2 additions & 2 deletions peg-macros/grammar.rustpeg
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,8 @@ use crate::tokens::FlatTokenStream;
use proc_macro2::{ TokenStream, Ident, Group, Literal, Delimiter, Span };

pub rule peg_grammar() -> Grammar
= doc:rust_doc_comment() visibility:rust_visibility() "grammar" name:IDENT() args:grammar_args() "for" input_type:$(rust_type()) "{" items:item()* "}"
{ Grammar { doc, visibility, name, args, input_type, items } }
= doc:rust_doc_comment() visibility:rust_visibility() "grammar" name:IDENT() ty_params:rust_ty_params()? args:grammar_args() "for" input_type:$(rust_type()) "{" items:item()* "}"
{ Grammar { doc, visibility, name, ty_params, args, input_type, items } }

rule grammar_args() -> Vec<(Ident, TokenStream)>
= "(" args:((i:IDENT() ":" t:$(rust_type()) { (i, t) })**",") ","? ")" { args }
Expand Down
32 changes: 21 additions & 11 deletions peg-macros/translate.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@ fn extra_args_call(grammar: &Grammar) -> TokenStream {
struct Context<'a> {
rules: &'a HashMap<String, &'a Rule>,
rules_from_args: HashSet<String>,
extra_ty_params: &'a Option<Vec<TokenStream>>,
extra_args_call: TokenStream,
extra_args_def: TokenStream,
}
Expand All @@ -62,6 +63,7 @@ pub(crate) fn compile_grammar(grammar: &Grammar) -> TokenStream {
let context = &Context {
rules: &analysis.rules,
rules_from_args: HashSet::new(),
extra_ty_params: &grammar.ty_params,
extra_args_call: extra_args_call(grammar),
extra_args_def: extra_args_def(grammar),
};
Expand Down Expand Up @@ -107,6 +109,7 @@ pub(crate) fn compile_grammar(grammar: &Grammar) -> TokenStream {

let doc = &grammar.doc;
let input_type = &grammar.input_type;
let input_ty_params = ty_params_slice(&grammar.ty_params);
let visibility = &grammar.visibility;

let mut errors = Vec::new();
Expand All @@ -124,8 +127,8 @@ pub(crate) fn compile_grammar(grammar: &Grammar) -> TokenStream {
#visibility mod #name {
#[allow(unused_imports)]
use super::*;
type Input = #input_type;
type PositionRepr = <Input as ::peg::Parse>::PositionRepr;
type Input<#(#input_ty_params),*> = #input_type;
type PositionRepr<#(#input_ty_params),*> = <Input<#(#input_ty_params),*> as ::peg::Parse>::PositionRepr;

#(#errors)*
#(#items)*
Expand Down Expand Up @@ -165,14 +168,19 @@ fn make_parse_state(grammar: &Grammar) -> TokenStream {
}
}

fn rule_params_list(rule: &Rule) -> Vec<TokenStream> {
fn ty_params_slice(ty_params: &Option<Vec<TokenStream>>) -> &[TokenStream] {
ty_params.as_ref().map(|x| &x[..]).unwrap_or(&[])
}

fn rule_params_list(context: &Context, rule: &Rule) -> Vec<TokenStream> {
let extra_ty_params = ty_params_slice(context.extra_ty_params);
let span = rule.span.resolved_at(Span::mixed_site());
rule.params.iter().map(|param| {
let name = &param.name;
match &param.ty {
RuleParamTy::Rust(ty) => quote_spanned!{ span => #name: #ty },
RuleParamTy::Rule(ty) => quote_spanned!{ span =>
#name: impl Fn(&'input Input, &mut ParseState<'input>, &mut ::peg::error::ErrorState, usize) -> ::peg::RuleResult<#ty>
#name: impl Fn(&'input Input<#(#extra_ty_params),*>, &mut ParseState<'input>, &mut ::peg::error::ErrorState, usize) -> ::peg::RuleResult<#ty>
},
}
}).collect()
Expand All @@ -184,7 +192,8 @@ fn compile_rule(context: &Context, rule: &Rule) -> TokenStream {
let name = format_ident!("__parse_{}", rule.name, span=span);
let ret_ty = rule.ret_type.clone().unwrap_or_else(|| quote!(()));
let result_used = rule.ret_type.is_some();
let ty_params = rule.ty_params.as_ref().map(|x| &x[..]).unwrap_or(&[]);
let ty_params = ty_params_slice(&rule.ty_params);
let extra_ty_params = ty_params_slice(context.extra_ty_params);

let mut context = context.clone();
context
Expand Down Expand Up @@ -217,7 +226,7 @@ fn compile_rule(context: &Context, rule: &Rule) -> TokenStream {

let extra_args_def = &context.extra_args_def;

let rule_params = rule_params_list(rule);
let rule_params = rule_params_list(&context, rule);

if rule.cached {
let cache_field = format_ident!("{}_cache", rule.name);
Expand All @@ -236,7 +245,7 @@ fn compile_rule(context: &Context, rule: &Rule) -> TokenStream {
};

quote_spanned! { span =>
fn #name<'input #(, #ty_params)*>(__input: &'input Input, __state: &mut ParseState<'input>, __err_state: &mut ::peg::error::ErrorState, __pos: usize #extra_args_def #(, #rule_params)*) -> ::peg::RuleResult<#ret_ty> {
fn #name<'input #(, #extra_ty_params)* #(, #ty_params)*>(__input: &'input Input<#(#extra_ty_params),*>, __state: &mut ParseState<'input>, __err_state: &mut ::peg::error::ErrorState, __pos: usize #extra_args_def #(, #rule_params)*) -> ::peg::RuleResult<#ret_ty> {
#![allow(non_snake_case, unused)]
if let Some(entry) = __state.#cache_field.get(&__pos) {
#cache_trace
Expand All @@ -249,7 +258,7 @@ fn compile_rule(context: &Context, rule: &Rule) -> TokenStream {
}
} else {
quote_spanned! { span =>
fn #name<'input #(, #ty_params)*>(__input: &'input Input, __state: &mut ParseState<'input>, __err_state: &mut ::peg::error::ErrorState, __pos: usize #extra_args_def #(, #rule_params)*) -> ::peg::RuleResult<#ret_ty> {
fn #name<'input #(, #extra_ty_params)* #(, #ty_params)*>(__input: &'input Input<#(#extra_ty_params),*>, __state: &mut ParseState<'input>, __err_state: &mut ::peg::error::ErrorState, __pos: usize #extra_args_def #(, #rule_params)*) -> ::peg::RuleResult<#ret_ty> {
#![allow(non_snake_case, unused)]
#wrapped_body
}
Expand All @@ -264,8 +273,9 @@ fn compile_rule_export(context: &Context, rule: &Rule) -> TokenStream {
let ret_ty = rule.ret_type.clone().unwrap_or_else(|| quote!(()));
let visibility = &rule.visibility;
let parse_fn = format_ident!("__parse_{}", rule.name.to_string(), span = name.span());
let ty_params = rule.ty_params.as_ref().map(|x| &x[..]).unwrap_or(&[]);
let rule_params = rule_params_list(rule);
let ty_params = ty_params_slice(&rule.ty_params);
let extra_ty_params = ty_params_slice(context.extra_ty_params);
let rule_params = rule_params_list(context, rule);
let rule_params_call: Vec<TokenStream> = rule
.params
.iter()
Expand All @@ -280,7 +290,7 @@ fn compile_rule_export(context: &Context, rule: &Rule) -> TokenStream {

quote_spanned! { span =>
#doc
#visibility fn #name<'input #(, #ty_params)*>(__input: &'input Input #extra_args_def #(, #rule_params)*) -> ::std::result::Result<#ret_ty, ::peg::error::ParseError<PositionRepr>> {
#visibility fn #name<'input #(, #extra_ty_params)* #(, #ty_params)*>(__input: &'input Input<#(#extra_ty_params),*> #extra_args_def #(, #rule_params)*) -> ::std::result::Result<#ret_ty, ::peg::error::ParseError<PositionRepr<#(#extra_ty_params),*>>> {
#![allow(non_snake_case, unused)]

let mut __err_state = ::peg::error::ErrorState::new(::peg::Parse::start(__input));
Expand Down
24 changes: 24 additions & 0 deletions tests/run-pass/lifetimes.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
#[derive(Clone)]
pub struct Token<'text>(&'text str);

peg::parser!{
grammar tokenparser<'t>() for [Token<'t>] {
pub rule program() -> Vec<&'t str> = list()

// add this indirection to ensure that rule args work with a global lifetime
rule commasep<T>(x: rule<T>) -> Vec<T> = v:(x() ** [Token(",")]) [Token(",")]? { v }

rule list() -> Vec<&'t str> = [Token("(")] l:commasep(<string()>) [Token(")")] { l }
rule string() -> &'t str = [Token(inner)] { inner }
}
}

fn main() {
let input = "(one,two)";
assert_eq!(
tokenparser::program(
&[Token(&input[0..1]), Token(&input[1..4]), Token(&input[4..5]), Token(&input[5..8]), Token(&input[8..9])],
),
Ok(vec!["one", "two"])
);
}