Fix interpreted mode not showing lexer tokens when the parse tree is …

…available

README.md

@@ -17,7 +17,7 @@ ANTLR grammar goes to the editor on the left, and the text to parse goes to the
 Output panel will display stream of tokens from lexer, parse tree (if available) and/or list of errors.
 
 Parse tree can be exported in [dot](https://en.wikipedia.org/wiki/DOT_(graph_description_language)) format to be rendered with [graphviz](https://graphviz.org/).
-Parsing must produce a parse tree before a Graph → Export menu could be available. 
+Parsing must complete successfully before a Graph → Export menu could be available. 
 
 ![](antlrscope.png)
 
@@ -32,3 +32,12 @@ The compiled mode generates Lexer and Parser source files, compiles them with Ja
 to tokenize and parse the code. It works exactly as if you generate a parser from the grammar as part of your maven/gradle build.
 
 The interpreted mode is faster and doesn't require additional disk space, so if you don't need predicates, you can use it. 
+
+## Limitations
+
+AntlrScope always uses Java as a target language for generated lexers and parsers. If your
+grammar includes code in other languages, you won't be able to generate lexer/parser and parse
+the text successfully. 
+
+You can try using interpreted mode, so that all the code in the grammar
+is ignored, but, obviously, this might produce results that are different from expected.

build.gradle

@@ -9,7 +9,7 @@ plugins {
 mainClassName = 'net.ninjacat.antlrscope.AppKt'
 
 group 'net.ninjacat'
-version '0.4.1'
+version '0.4.2'
 
 repositories {
     mavenCentral()

src/main/kotlin/net/ninjacat/antlrscope/antlr/AntlrCompiler.kt

@@ -44,7 +44,7 @@ class AntlrCompiler(grammar: String, text: String, private val javaCompiler: Jav
                     val runner = AntlrExecutor(text, packageName, grammarName, path.parent.resolve("classes"))
                     val results = runner.parse(errorListener)
                     tokens = results.tokens!!
-                    tree = if (results.tree != null) convertParseTree(results.tree!!, results.ruleNames) else null
+                    tree = if (results.tree != null) convertParseTree(results.tree, results.ruleNames) else null
                     ruleNames = results.ruleNames
 
                 }

src/main/kotlin/net/ninjacat/antlrscope/antlr/AntlrInterpreter.kt

@@ -4,6 +4,7 @@ import org.antlr.runtime.ANTLRStringStream
 import org.antlr.v4.Tool
 import org.antlr.v4.runtime.CharStreams
 import org.antlr.v4.runtime.CommonTokenStream
+import org.antlr.v4.runtime.ListTokenSource
 import org.antlr.v4.tool.Grammar
 import org.antlr.v4.tool.ast.GrammarRootAST
 
@@ -14,22 +15,18 @@ class AntlrInterpreter(grammar: String, text: String): AntlrGrammarParser(gramma
         try {
             ruleNames = antlrGrammar!!.ruleNames
             val lexEngine = antlrGrammar!!.createLexerInterpreter(CharStreams.fromString(text))
+            val lexTokens = lexEngine.allTokens
+            tokens = convertTokens(lexEngine, lexTokens)
             val errorListener = ErrorListener(errors)
             lexEngine.addErrorListener(errorListener)
             if (antlrGrammar!!.isCombined) {
-                val tokens = CommonTokenStream(lexEngine)
-                val parser = antlrGrammar!!.createParserInterpreter(tokens)
+                val tokenStream = CommonTokenStream(ListTokenSource(lexTokens))
+                val parser = antlrGrammar!!.createParserInterpreter(tokenStream)
                 parser.addErrorListener(errorListener)
-                tree = if (tree != null) {
-                    convertParseTree(parser.parse(antlrGrammar!!.getRule(0).index), antlrGrammar?.ruleNames!!)
-                } else {
-                    null
-                }
+                tree = convertParseTree(parser.parse(antlrGrammar!!.getRule(0).index), antlrGrammar?.ruleNames!!)
                 errors.addAll(errorListener.errors)
             } else {
-                val tokens = convertTokens(lexEngine, lexEngine.allTokens)
                 tree = null
-                this.tokens = tokens
             }
             return true
         } catch (ex: Exception) {