diff --git a/.bazelrc b/.bazelrc index 163ffecbf..6376aaaab 100644 --- a/.bazelrc +++ b/.bazelrc @@ -102,3 +102,10 @@ coverage --test_tag_filters=-no-coverage # https://github.com/bazelbuild/bazel/issues/4867#issuecomment-830402410 common:quiet --ui_event_filters=-info,-stderr common:quiet --noshow_progress + +# Use --config=fail-fast to speed up local iteration on tests by letting bazel test stop right on +# the first failing test. +# Let bazel test stop on the first failing test target. +test:fail-fast --notest_keep_going +# Instruct test runners to fail a test target on the first failing test. +test:fail-fast --test_runner_fail_fast diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a8c76d29e..ae5a296ac 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -71,6 +71,9 @@ To run the tests, execute the following command: $ bazel test //... ``` +If you are bisecting a bug or otherwise want test execution to stop right after the first failure, use `--config=fail-fast`. +This is especially useful with long-running or parameterized tests. + #### Debugging If you need to debug an issue that can only be reproduced by an integration test (`java_fuzz_target_test`), you can start Jazzer in debug mode via `--config=debug`. diff --git a/WORKSPACE.bazel b/WORKSPACE.bazel index aa4b7a6ca..4a3f75e9a 100644 --- a/WORKSPACE.bazel +++ b/WORKSPACE.bazel @@ -87,6 +87,12 @@ http_archive( http_archive( name = "contrib_rules_jvm", + patch_args = ["-p1"], + patches = [ + # Add support for --test_runner_fail_fast. + # https://github.com/bazel-contrib/rules_jvm/pull/221 + "//third_party:rules_jvm-fail-fast.patch", + ], sha256 = "4d62589dc6a55e74bbe33930b826d593367fc777449a410604b2ad7c6c625ef7", strip_prefix = "rules_jvm-0.19.0", url = "https://github.com/bazel-contrib/rules_jvm/releases/download/v0.19.0/rules_jvm-v0.19.0.tar.gz", diff --git a/third_party/rules_jvm-fail-fast.patch b/third_party/rules_jvm-fail-fast.patch new file mode 100644 index 000000000..45c6dbf14 --- /dev/null +++ b/third_party/rules_jvm-fail-fast.patch @@ -0,0 +1,149 @@ +From c7153ed03cfe57a956d366fe9c6dfd844202a121 Mon Sep 17 00:00:00 2001 +From: Fabian Meumertzheim +Date: Thu, 9 Nov 2023 13:37:47 +0100 +Subject: [PATCH] junit5: Add support for `--test_runner_fail_fast` + +When the Bazel flag `--test_runner_fail_fast` is enabled, the JUnit 5 +runner will now skip all other tests after the first one has failed in +an attempt to speed up the (failing) test run. This is useful in some +CI setups as well as during bisection. +--- + .../junit5/ActualRunner.java | 8 +++- + .../contrib_rules_jvm/junit5/BUILD.bazel | 7 +++ + .../junit5/FailFastExtension.java | 44 +++++++++++++++++++ + .../junit5/JUnit5Runner.java | 3 ++ + .../org.junit.jupiter.api.extension.Extension | 1 + + 5 files changed, 61 insertions(+), 2 deletions(-) + create mode 100644 java/src/com/github/bazel_contrib/contrib_rules_jvm/junit5/FailFastExtension.java + create mode 100644 java/src/com/github/bazel_contrib/contrib_rules_jvm/junit5/META-INF/services/org.junit.jupiter.api.extension.Extension + +diff --git a/java/src/com/github/bazel_contrib/contrib_rules_jvm/junit5/ActualRunner.java b/java/src/com/github/bazel_contrib/contrib_rules_jvm/junit5/ActualRunner.java +index 660916cd..71389540 100644 +--- a/java/src/com/github/bazel_contrib/contrib_rules_jvm/junit5/ActualRunner.java ++++ b/java/src/com/github/bazel_contrib/contrib_rules_jvm/junit5/ActualRunner.java +@@ -20,6 +20,7 @@ + import java.util.Arrays; + import java.util.List; + import java.util.stream.Collectors; ++import org.junit.jupiter.engine.Constants; + import org.junit.platform.engine.DiscoverySelector; + import org.junit.platform.engine.discovery.DiscoverySelectors; + import org.junit.platform.launcher.Launcher; +@@ -44,10 +45,11 @@ public boolean run(String testClassName) { + + try (BazelJUnitOutputListener bazelJunitXml = new BazelJUnitOutputListener(xmlOut)) { + CommandLineSummary summary = new CommandLineSummary(); ++ FailFastExtension failFastExtension = new FailFastExtension(); + + LauncherConfig config = + LauncherConfig.builder() +- .addTestExecutionListeners(bazelJunitXml, summary) ++ .addTestExecutionListeners(bazelJunitXml, summary, failFastExtension) + .addPostDiscoveryFilters(TestSharding.makeShardFilter()) + .build(); + +@@ -74,7 +76,9 @@ public boolean run(String testClassName) { + LauncherDiscoveryRequestBuilder.request() + .selectors(classSelectors) + .configurationParameter(LauncherConstants.CAPTURE_STDERR_PROPERTY_NAME, "true") +- .configurationParameter(LauncherConstants.CAPTURE_STDOUT_PROPERTY_NAME, "true"); ++ .configurationParameter(LauncherConstants.CAPTURE_STDOUT_PROPERTY_NAME, "true") ++ .configurationParameter( ++ Constants.EXTENSIONS_AUTODETECTION_ENABLED_PROPERTY_NAME, "true"); + + String filter = System.getenv("TESTBRIDGE_TEST_ONLY"); + request.filters(new PatternFilter(filter)); +diff --git a/java/src/com/github/bazel_contrib/contrib_rules_jvm/junit5/BUILD.bazel b/java/src/com/github/bazel_contrib/contrib_rules_jvm/junit5/BUILD.bazel +index 59390d64..97914014 100644 +--- a/java/src/com/github/bazel_contrib/contrib_rules_jvm/junit5/BUILD.bazel ++++ b/java/src/com/github/bazel_contrib/contrib_rules_jvm/junit5/BUILD.bazel +@@ -58,11 +58,18 @@ java_library( + "--release", + "8", + ], ++ resource_strip_prefix = package_name(), ++ resources = ["META-INF/services/org.junit.jupiter.api.extension.Extension"], + deps = [ + ":system-exit-toggle", + # The only dependencies here are those required to run + # a junit5 test. We try not to pollute the classpath, so + # be very careful when adding new deps here. ++ # ++ # junit-jupiter-api is only a compilation dependency and not listed in JUNIT5_DEPS. This is ++ # fine as it is a transitive dependency of junit-jupiter-engine and will thus be available ++ # at runtime. ++ artifact("org.junit.jupiter:junit-jupiter-api"), + artifact("org.junit.jupiter:junit-jupiter-engine"), + artifact("org.junit.platform:junit-platform-commons"), + artifact("org.junit.platform:junit-platform-engine"), +diff --git a/java/src/com/github/bazel_contrib/contrib_rules_jvm/junit5/FailFastExtension.java b/java/src/com/github/bazel_contrib/contrib_rules_jvm/junit5/FailFastExtension.java +new file mode 100644 +index 00000000..458d3bf9 +--- /dev/null ++++ b/java/src/com/github/bazel_contrib/contrib_rules_jvm/junit5/FailFastExtension.java +@@ -0,0 +1,44 @@ ++package com.github.bazel_contrib.contrib_rules_jvm.junit5; ++ ++import java.util.concurrent.atomic.AtomicBoolean; ++import org.junit.jupiter.api.extension.ConditionEvaluationResult; ++import org.junit.jupiter.api.extension.ExecutionCondition; ++import org.junit.jupiter.api.extension.ExtensionContext; ++import org.junit.platform.engine.TestExecutionResult; ++import org.junit.platform.engine.TestExecutionResult.Status; ++import org.junit.platform.launcher.TestExecutionListener; ++import org.junit.platform.launcher.TestIdentifier; ++ ++public class FailFastExtension implements ExecutionCondition, TestExecutionListener { ++ /** ++ * This environment variable is set to 1 if Bazel is run with --test_runner_fail_fast, indicating ++ * that the test runner should exit as soon as possible after the first failure. {@see ++ * https://github.com/bazelbuild/bazel/commit/957554037ced26dc1860b9c23445a8ccc44f697e} ++ */ ++ private static final boolean SHOULD_FAIL_FAST = ++ "1".equals(System.getenv("TESTBRIDGE_TEST_RUNNER_FAIL_FAST")); ++ ++ private static final AtomicBoolean SOME_TEST_FAILED = new AtomicBoolean(); ++ ++ @Override ++ public ConditionEvaluationResult evaluateExecutionCondition(ExtensionContext extensionContext) { ++ if (!SHOULD_FAIL_FAST) { ++ return ConditionEvaluationResult.enabled( ++ "Running test since --test_runner_fail_fast is not enabled"); ++ } ++ if (SOME_TEST_FAILED.get()) { ++ return ConditionEvaluationResult.disabled( ++ "Skipping test since --test_runner_fail_fast is enabled and another test has failed"); ++ } else { ++ return ConditionEvaluationResult.enabled("Running test since no other test has failed yet"); ++ } ++ } ++ ++ @Override ++ public void executionFinished( ++ TestIdentifier testIdentifier, TestExecutionResult testExecutionResult) { ++ if (SHOULD_FAIL_FAST && testExecutionResult.getStatus().equals(Status.FAILED)) { ++ SOME_TEST_FAILED.set(true); ++ } ++ } ++} +diff --git a/java/src/com/github/bazel_contrib/contrib_rules_jvm/junit5/JUnit5Runner.java b/java/src/com/github/bazel_contrib/contrib_rules_jvm/junit5/JUnit5Runner.java +index b1e437b2..e7ce869b 100644 +--- a/java/src/com/github/bazel_contrib/contrib_rules_jvm/junit5/JUnit5Runner.java ++++ b/java/src/com/github/bazel_contrib/contrib_rules_jvm/junit5/JUnit5Runner.java +@@ -91,6 +91,9 @@ private static SystemExitToggle getSystemExitToggle() { + } + + private static void detectJUnit5Classes() { ++ checkClass( ++ "org.junit.jupiter.api.extension.ExecutionCondition", ++ "org.junit.jupiter:junit-jupiter-api"); + checkClass( + "org.junit.jupiter.engine.JupiterTestEngine", "org.junit.jupiter:junit-jupiter-engine"); + checkClass( +diff --git a/java/src/com/github/bazel_contrib/contrib_rules_jvm/junit5/META-INF/services/org.junit.jupiter.api.extension.Extension b/java/src/com/github/bazel_contrib/contrib_rules_jvm/junit5/META-INF/services/org.junit.jupiter.api.extension.Extension +new file mode 100644 +index 00000000..6bd285fe +--- /dev/null ++++ b/java/src/com/github/bazel_contrib/contrib_rules_jvm/junit5/META-INF/services/org.junit.jupiter.api.extension.Extension +@@ -0,0 +1 @@ ++com.github.bazel_contrib.contrib_rules_jvm.junit5.FailFastExtension