Skip to content

Commit

Permalink
refactor: parser primary expr loc, ast pretty printer and add more te…
Browse files Browse the repository at this point in the history 
…sts (#558)

refactor: parser primary expr loc, ast pretty printer and add more tests.
  • Loading branch information
@Peefy
Peefy committed May 24, 2023
1 parent a55c820 commit 7d26909
Show file tree
Hide file tree
Showing 31 changed files with 150 additions and 121 deletions.
32 changes: 21 additions & 11 deletions kclvm/ast_pretty/src/node.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,10 @@ impl<'p, 'ctx> MutSelfTypedResultWalker<'ctx> for Printer<'p> {
for comment in &module.comments {
self.comments.push_back(comment.clone());
}
if !module.doc.is_empty() {
self.write(&module.doc);
self.write_newline();
}
self.stmts(&module.body);
}

Expand Down Expand Up @@ -443,11 +447,12 @@ impl<'p, 'ctx> MutSelfTypedResultWalker<'ctx> for Printer<'p> {
.map(|e| e.line)
.collect::<HashSet<u64>>();
// There are comments in the configuration block.
let has_comment = list_expr
.elts
.iter()
.map(|e| self.has_comments_on_node(e))
.all(|r| r);
let has_comment = !list_expr.elts.is_empty()
&& list_expr
.elts
.iter()
.map(|e| self.has_comments_on_node(e))
.all(|r| r);
// When there are comments in the configuration block, print them as multiline configurations.
let mut in_one_line = line_set.len() <= 1 && !has_comment;
if let Some(elt) = list_expr.elts.first() {
Expand Down Expand Up @@ -605,11 +610,12 @@ impl<'p, 'ctx> MutSelfTypedResultWalker<'ctx> for Printer<'p> {
fn walk_config_expr(&mut self, config_expr: &'ctx ast::ConfigExpr) -> Self::Result {
let line_set: HashSet<u64> = config_expr.items.iter().map(|item| item.line).collect();
// There are comments in the configuration block.
let has_comment = config_expr
.items
.iter()
.map(|item| self.has_comments_on_node(item))
.all(|r| r);
let has_comment = !config_expr.items.is_empty()
&& config_expr
.items
.iter()
.map(|item| self.has_comments_on_node(item))
.all(|r| r);
// When there are comments in the configuration block, print them as multiline configurations.
let mut in_one_line = line_set.len() <= 1 && !has_comment;
// When there are complex configuration blocks in the configuration block, print them as multiline configurations.
Expand Down Expand Up @@ -729,10 +735,14 @@ impl<'p, 'ctx> MutSelfTypedResultWalker<'ctx> for Printer<'p> {
}

fn walk_number_lit(&mut self, number_lit: &'ctx ast::NumberLit) -> Self::Result {
match number_lit.value {
match &number_lit.value {
ast::NumberLitValue::Int(int_val) => self.write(&int_val.to_string()),
ast::NumberLitValue::Float(float_val) => self.write(&float_val.to_string()),
}
// Number suffix e.g., 1Gi
if let Some(binary_suffix) = &number_lit.binary_suffix {
self.write(&binary_suffix.value())
}
}

fn walk_string_lit(&mut self, string_lit: &'ctx ast::StringLit) -> Self::Result {
Expand Down
9 changes: 7 additions & 2 deletions kclvm/ast_pretty/src/test_data/codelayout.input
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,6 @@

"""
Module documents
"""


import math as alias_math
Expand Down Expand Up @@ -73,4 +75,7 @@ aa = 1
assert aa == 1,"message"
assert aa == 1 if aa,"message"
aaaa = (1 + 2 / 2) if _a == 2 + + 134.3 else ("a"*3)
bbbb = "{}". format(a)
bbbb = "{}". format(a)
empty_list = []
empty_config = {}
number_suffix = 1Gi
6 changes: 6 additions & 0 deletions kclvm/ast_pretty/src/test_data/codelayout.output
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,6 @@
"""
Module documents
"""
import math as alias_math
schema Person(Base):
name: str
Expand Down Expand Up @@ -77,3 +80,6 @@ assert aa == 1, "message"
assert aa == 1 if aa, "message"
aaaa = (1 + 2 / 2) if _a == 2 + +134.3 else ("a" * 3)
bbbb = "{}".format(a)
empty_list = []
empty_config = {}
number_suffix = 1Gi
9 changes: 9 additions & 0 deletions kclvm/ast_pretty/src/test_data/lambda.input
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
sumFunc1 = lambda x, y {
# Inline comments 1
z = x + y
# Inline comments 2
z + x
}
sumFunc2 = lambda x, y = 1 {
Expand All @@ -18,3 +20,10 @@ x0 = sumFunc1(1, 2)
x1 = sumFunc1(2, 3)
x2 = sumFunc1(3, 4)
x3 = sumFunc1(4, 5)

x = lambda {
# Inline comments
1

}()

7 changes: 7 additions & 0 deletions kclvm/ast_pretty/src/test_data/lambda.output
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
sumFunc1 = lambda x, y {
# Inline comments 1
z = x + y
# Inline comments 2
z + x

}
Expand All @@ -19,3 +21,8 @@ x0 = sumFunc1(1, 2)
x1 = sumFunc1(2, 3)
x2 = sumFunc1(3, 4)
x3 = sumFunc1(4, 5)
x = lambda {
# Inline comments
1

}()
30 changes: 14 additions & 16 deletions kclvm/parser/src/parser/expr.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -236,25 +236,26 @@ impl<'a> Parser<'a> {
/// primary_expr: operand | primary_expr select_suffix | primary_expr call_suffix | primary_expr slice_suffix
/// Note: we need to look ahead 2 tokens to match select_suffix and slice_suffix, which actually breaks LL1 rule.
fn parse_primary_expr(&mut self) -> NodeRef<Expr> {
let lo = self.token;
let mut operand = self.parse_operand_expr();

loop {
match self.token.kind {
TokenKind::Dot => {
// select_suffix
operand = self.parse_selector_expr(operand)
operand = self.parse_selector_expr(operand, lo)
}
TokenKind::Question => {
match self.cursor.peek() {
Some(token) => {
match token.kind {
TokenKind::Dot => {
// select_suffix
operand = self.parse_selector_expr(operand)
operand = self.parse_selector_expr(operand, lo)
}
TokenKind::OpenDelim(DelimToken::Bracket) => {
// slice_suffix
operand = self.parse_subscript_expr(operand)
operand = self.parse_subscript_expr(operand, lo)
}
_ => break operand,
}
Expand All @@ -266,11 +267,11 @@ impl<'a> Parser<'a> {
match dt {
DelimToken::Paren => {
// call_suffix
operand = self.parse_call_expr(operand)
operand = self.parse_call_expr(operand, lo)
}
DelimToken::Bracket => {
// slice_suffix
operand = self.parse_subscript_expr(operand)
operand = self.parse_subscript_expr(operand, lo)
}
_ => break operand,
}
Expand Down Expand Up @@ -301,8 +302,7 @@ impl<'a> Parser<'a> {

/// Syntax:
/// select_suffix: [QUESTION] DOT NAME
fn parse_selector_expr(&mut self, value: NodeRef<Expr>) -> NodeRef<Expr> {
let token = self.token;
fn parse_selector_expr(&mut self, value: NodeRef<Expr>, lo: token::Token) -> NodeRef<Expr> {
let has_question = match self.token.kind {
TokenKind::Question => {
self.bump();
Expand All @@ -320,18 +320,17 @@ impl<'a> Parser<'a> {
has_question,
ctx: ExprContext::Load,
}),
self.sess.struct_token_loc(token, self.prev_token),
self.sess.struct_token_loc(lo, self.prev_token),
))
}

/// Syntax:
/// call_suffix: LEFT_PARENTHESES [arguments [COMMA]] RIGHT_PARENTHESES
fn parse_call_expr(&mut self, func: NodeRef<Expr>) -> NodeRef<Expr> {
let token = self.token;
fn parse_call_expr(&mut self, func: NodeRef<Expr>, lo: token::Token) -> NodeRef<Expr> {
let call_expr = self.parse_call(func);
Box::new(Node::node(
Expr::Call(call_expr),
self.sess.struct_token_loc(token, self.prev_token),
self.sess.struct_token_loc(lo, self.prev_token),
))
}

Expand Down Expand Up @@ -375,8 +374,7 @@ impl<'a> Parser<'a> {

/// Syntax:
/// slice_suffix: [QUESTION] LEFT_BRACKETS (expr | [expr] COLON [expr] [COLON [expr]]) RIGHT_BRACKETS
fn parse_subscript_expr(&mut self, value: NodeRef<Expr>) -> NodeRef<Expr> {
let token = self.token;
fn parse_subscript_expr(&mut self, value: NodeRef<Expr>, lo: token::Token) -> NodeRef<Expr> {
let mut has_question = false;
// [QUESTION]
if self.token.kind == TokenKind::Question {
Expand Down Expand Up @@ -464,7 +462,7 @@ impl<'a> Parser<'a> {
ctx: ExprContext::Load,
has_question,
}),
self.sess.struct_token_loc(token, self.prev_token),
self.sess.struct_token_loc(lo, self.prev_token),
))
} else {
if exprs[0].is_none() {
Expand All @@ -488,7 +486,7 @@ impl<'a> Parser<'a> {
ctx: ExprContext::Load,
has_question,
}),
self.sess.struct_token_loc(token, self.prev_token),
self.sess.struct_token_loc(lo, self.prev_token),
))
}
}
Expand Down Expand Up @@ -2043,7 +2041,7 @@ impl<'a> Parser<'a> {
TokenKind::OpenDelim(DelimToken::Paren) => {
self.bump();

self.parse_call_expr(func)
self.parse_call_expr(func, token)
}
_ => Box::new(Node::node(
Expr::Call(CallExpr {
Expand Down
23 changes: 15 additions & 8 deletions kclvm/parser/src/parser/module.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,17 +19,24 @@ impl<'a> Parser<'a> {
}
}

fn parse_doc(&mut self) -> String {
if let TokenKind::Literal(lit) = self.token.kind {
if let LitKind::Str { is_long_string, .. } = lit.kind {
if is_long_string {
let doc = format!("{:?}", self.token);
self.bump();
return doc;
pub(crate) fn parse_doc(&mut self) -> String {
// doc string
match self.token.kind {
TokenKind::Literal(lit) => {
if let LitKind::Str { .. } = lit.kind {
let doc_expr = self.parse_str_expr(lit);
self.skip_newlines();
match &doc_expr.node {
Expr::StringLit(str) => str.raw_value.clone(),
Expr::JoinedString(str) => str.raw_value.clone(),
_ => "".to_string(),
}
} else {
"".to_string()
}
}
_ => "".to_string(),
}
"".to_string()
}

fn parse_body(&mut self) -> Vec<NodeRef<Stmt>> {
Expand Down
17 changes: 1 addition & 16 deletions kclvm/parser/src/parser/stmt.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -856,22 +856,7 @@ impl<'a> Parser<'a> {
self.bump_token(TokenKind::Indent);

// doc string
let body_doc = match self.token.kind {
TokenKind::Literal(lit) => {
if let LitKind::Str { .. } = lit.kind {
let doc_expr = self.parse_str_expr(lit);
self.skip_newlines();
match &doc_expr.node {
Expr::StringLit(str) => str.raw_value.clone(),
Expr::JoinedString(str) => str.raw_value.clone(),
_ => "".to_string(),
}
} else {
"".to_string()
}
}
_ => "".to_string(),
};
let body_doc = self.parse_doc();

// mixin
let body_mixins = if self.token.is_keyword(kw::Mixin) {
Expand Down
Loading

0 comments on commit 7d26909

Please sign in to comment.