Clarify message of failed runtime tests

Signed-off-by: Ivan Kochurkin <[email protected]>
antlr · Jun 19, 2022 · b0f4531 · b0f4531
1 parent bc2f181
commit b0f4531
Show file tree

Hide file tree

Showing 3 changed files with 23 additions and 15 deletions.
diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/Processor.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/Processor.java
@@ -19,11 +19,11 @@ public class Processor {
 
 	public static ProcessorResult run(String[] arguments, String workingDirectory, Map<String, String> environmentVariables
 	) throws InterruptedException, IOException {
-		return new Processor(arguments, workingDirectory, environmentVariables, true).Start();
+		return new Processor(arguments, workingDirectory, environmentVariables, true).start();
 	}
 
 	public static ProcessorResult run(String[] arguments, String workingDirectory) throws InterruptedException, IOException {
-		return new Processor(arguments, workingDirectory, new HashMap<>(), true).Start();
+		return new Processor(arguments, workingDirectory, new HashMap<>(), true).start();
 	}
 
 	public Processor(String[] arguments, String workingDirectory, Map<String, String> environmentVariables,
@@ -34,7 +34,7 @@ public Processor(String[] arguments, String workingDirectory, Map<String, String
 		this.throwOnNonZeroErrorCode = throwOnNonZeroErrorCode;
 	}
 
-	public ProcessorResult Start() throws InterruptedException, IOException {
+	public ProcessorResult start() throws InterruptedException, IOException {
 		ProcessBuilder builder = new ProcessBuilder(arguments);
 		if (workingDirectory != null) {
 			builder.directory(new File(workingDirectory));

diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/RuntimeRunner.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/RuntimeRunner.java
@@ -63,7 +63,7 @@ public final String getTempDirPath() {
 		return tempTestDir.toString();
 	}
 
-	private final boolean saveTestDir;
+	private boolean saveTestDir;
 
 	protected final Path tempTestDir;
 
@@ -82,6 +82,10 @@ protected RuntimeRunner(Path tempDir, boolean saveTestDir) {
 		this.saveTestDir = saveTestDir;
 	}
 
+	public void setSaveTestDir(boolean saveTestDir) {
+		this.saveTestDir = saveTestDir;
+	}
+
 	public void close() {
 		removeTempTestDirIfRequired();
 	}

diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/RuntimeTests.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/RuntimeTests.java
@@ -98,7 +98,11 @@ public List<DynamicNode> runtimeTests() {
 			for (RuntimeTestDescriptor descriptor : descriptors) {
 				descriptorTests.add(dynamicTest(descriptor.name, () -> {
 					try (RuntimeRunner runner = createRuntimeRunner()) {
-						test(descriptor, runner);
+						String errorMessage = test(descriptor, runner);
+						if (errorMessage != null) {
+							runner.setSaveTestDir(true);
+							fail(descriptor.name + "; " + errorMessage + "\nDirectory: " + runner.getTempDirPath());
+						}
 					}
 				}));
 			}
@@ -108,11 +112,11 @@ public List<DynamicNode> runtimeTests() {
 		return result;
 	}
 
-	private static void test(RuntimeTestDescriptor descriptor, RuntimeRunner runner) {
+	private static String test(RuntimeTestDescriptor descriptor, RuntimeRunner runner) {
 		String targetName = runner.getLanguage();
 		if (descriptor.ignore(targetName)) {
 			System.out.println("Ignore " + descriptor);
-			return;
+			return null;
 		}
 
 		FileUtils.mkdir(runner.getTempDirPath());
@@ -183,21 +187,19 @@ private static void test(RuntimeTestDescriptor descriptor, RuntimeRunner runner)
 
 		State result = runner.run(runOptions);
 
-		assertCorrectOutput(descriptor, targetName, result);
+		return assertCorrectOutput(descriptor, targetName, result);
 	}
 
-	private static void assertCorrectOutput(RuntimeTestDescriptor descriptor, String targetName, State state) {
+	private static String assertCorrectOutput(RuntimeTestDescriptor descriptor, String targetName, State state) {
 		ExecutedState executedState;
 		if (state instanceof ExecutedState) {
 			executedState = (ExecutedState)state;
 			if (executedState.exception != null) {
-				fail(state.getErrorMessage());
-				return;
+				return state.getErrorMessage();
 			}
 		}
 		else {
-			fail(state.getErrorMessage());
-			return;
+			return state.getErrorMessage();
 		}
 
 		String expectedOutput = descriptor.output;
@@ -214,10 +216,12 @@ private static void assertCorrectOutput(RuntimeTestDescriptor descriptor, String
 						"expectedOutput:<" + expectedOutput + ">; actualOutput:<" + executedState.output + ">; ";
 			}
 
-			fail("[" + targetName + ":" + descriptor.name + "] " +
+			return "[" + targetName + ":" + descriptor.name + "] " +
 					message +
 					"expectedParseErrors:<" + expectedParseErrors + ">;" +
-					"actualParseErrors:<" + executedState.errors + ">.");
+					"actualParseErrors:<" + executedState.errors + ">.";
 		}
+
+		return null;
 	}
 }