Skip to content

Commit

Permalink
Avoid tokenizer exceptions for qualified expressions
Browse files Browse the repository at this point in the history
  • Loading branch information
Paebbels committed Jul 8, 2023
2 parents 42ce6d0 + 41daf49 commit 275a778
Show file tree
Hide file tree
Showing 2 changed files with 66 additions and 1 deletion.
8 changes: 7 additions & 1 deletion pyVHDLParser/Token/Parser.py
Original file line number Diff line number Diff line change
Expand Up @@ -335,7 +335,13 @@ def GetVHDLTokenizer(cls, iterable: Iterator[str]):
elif tokenKind is cls.TokenKind.PossibleCharacterLiteral:
buffer += char
if len(buffer) == 2:
if buffer[1] == "'":
if buffer[1] == "(" and isinstance(previousToken, WordToken):
previousToken = CharacterToken(previousToken, "'", start)
yield previousToken
previousToken = CharacterToken(previousToken, "(", SourceCodePosition(row, column, absolute))
yield previousToken
tokenKind = cls.TokenKind.OtherChars
elif buffer[1] == "'":
previousToken = CharacterToken(previousToken, "'", start)
yield previousToken
previousToken = CharacterToken(previousToken, "'", SourceCodePosition(row, column, absolute))
Expand Down
59 changes: 59 additions & 0 deletions tests/unit/Tokenizer/Tokens.py
Original file line number Diff line number Diff line change
Expand Up @@ -256,6 +256,7 @@ class Sequence_5(TestCase, ExpectedDataMixin, TokenSequence):
]
)


class Sequence_6(TestCase, ExpectedDataMixin, TokenSequence):
code = """if Clk'event and Clk = '1' then -- rising clock edge\nname'attr1'attr2\nsignal reg_catch1 : std_logic_vector(flags_src1'range);"""
tokenStream = ExpectedTokenStream(
Expand Down Expand Up @@ -301,6 +302,64 @@ class Sequence_6(TestCase, ExpectedDataMixin, TokenSequence):
)


class Sequence_7(TestCase, ExpectedDataMixin, TokenSequence):
code = """constant BIT_STRING : UNSIGNED(0 downto 0) := UNSIGNED'(x\"0\");\nconstant LPAREN_CHAR : character := '(';\nfoo'('0')\nbar'('(')"""
tokenStream = ExpectedTokenStream(
[ (StartOfDocumentToken, None),
(WordToken, "constant"),
(WhitespaceToken, " "),
(WordToken, "BIT_STRING"),
(WhitespaceToken, " "),
(CharacterToken, ":"),
(WhitespaceToken, " "),
(WordToken, "UNSIGNED"),
(CharacterToken, "("),
(IntegerLiteralToken, "0"),
(WhitespaceToken, " "),
(WordToken, "downto"),
(WhitespaceToken, " "),
(IntegerLiteralToken, "0"),
(CharacterToken, ")"),
(WhitespaceToken, " "),
(FusedCharacterToken, ":="),
(WhitespaceToken, " "),
(WordToken, "UNSIGNED"),
(CharacterToken, "'"),
(CharacterToken, "("),
(WordToken, "x"),
(StringLiteralToken, "0"),
(CharacterToken, ")"),
(CharacterToken, ";"),
(LinebreakToken, None),
(WordToken, "constant"),
(WhitespaceToken, " "),
(WordToken, "LPAREN_CHAR"),
(WhitespaceToken, " "),
(CharacterToken, ":"),
(WhitespaceToken, " "),
(WordToken, "character"),
(WhitespaceToken, " "),
(FusedCharacterToken, ":="),
(WhitespaceToken, " "),
(CharacterLiteralToken, "("),
(CharacterToken, ";"),
(LinebreakToken, None),
(WordToken, "foo"),
(CharacterToken, "'"),
(CharacterToken, "("),
(CharacterLiteralToken, "0"),
(CharacterToken, ")"),
(LinebreakToken, None),
(WordToken, "bar"),
(CharacterToken, "'"),
(CharacterToken, "("),
(CharacterLiteralToken, "("),
(CharacterToken, ")"),
(EndOfDocumentToken, None)
]
)


class Tokenizer_ExceptionInKeyword(TestCase, ExpectedDataMixin, TokenSequence):
code = """keyword"""
tokenStream = ExpectedTokenStream(
Expand Down

0 comments on commit 275a778

Please sign in to comment.