diff --git a/README.adoc b/README.adoc index f75c6fa3..172c3f3d 100644 --- a/README.adoc +++ b/README.adoc @@ -57,7 +57,9 @@ The `retry` extension is of the following type: ---- package org.gradle.testretry; +import org.gradle.api.Action; import org.gradle.api.provider.Property; +import org.gradle.api.provider.SetProperty; import org.gradle.api.tasks.testing.Test; /** @@ -67,6 +69,11 @@ import org.gradle.api.tasks.testing.Test; */ public interface TestRetryTaskExtension { + /** + * The name of the extension added to each test task. + */ + String NAME = "retry"; + /** * Whether tests that initially fail and then pass on retry should fail the task. *

@@ -75,7 +82,7 @@ public interface TestRetryTaskExtension { *

* This setting has no effect if {@link Test#getIgnoreFailures()} is set to true. * - * @return whether tests that initially fail and then pass on retry should fail the task + * @return whether tests that initially fails and then pass on retry should fail the task */ Property getFailOnPassedAfterRetry(); @@ -104,6 +111,77 @@ public interface TestRetryTaskExtension { */ Property getMaxFailures(); + /** + * The filter for specifying which tests may be retried. + */ + Filter getFilter(); + + /** + * The filter for specifying which tests may be retried. + */ + void filter(Action action); + + /** + * A filter for specifying which tests may be retried. + * + * By default, all tests are eligible for retrying. + */ + interface Filter { + + /** + * The patterns used to include tests based on their class name. + * + * The pattern string matches against qualified class names. + * It may contain '*' characters, which match zero or more of any character. + * + * A class name only has to match one pattern to be included. + * + * If no patterns are specified, all classes (that also meet other configured filters) will be included. + */ + SetProperty getIncludeClasses(); + + /** + * The patterns used to include tests based on their class level annotations. + * + * The pattern string matches against the qualified class names of a test class's annotations. + * It may contain '*' characters, which match zero or more of any character. + * + * A class need only have one annotation matching any of the patterns to be included. + * + * Annotations present on super classes that are {@code @Inherited} are considered when inspecting subclasses. + * + * If no patterns are specified, all classes (that also meet other configured filters) will be included. + */ + SetProperty getIncludeAnnotationClasses(); + + /** + * The patterns used to exclude tests based on their class name. + * + * The pattern string matches against qualified class names. + * It may contain '*' characters, which match zero or more of any character. + * + * A class name only has to match one pattern to be excluded. + * + * If no patterns are specified, all classes (that also meet other configured filters) will be included. + */ + SetProperty getExcludeClasses(); + + /** + * The patterns used to exclude tests based on their class level annotations. + * + * The pattern string matches against the qualified class names of a test class's annotations. + * It may contain '*' characters, which match zero or more of any character. + * + * A class need only have one annotation matching any of the patterns to be excluded. + * + * Annotations present on super classes that are {@code @Inherited} are considered when inspecting subclasses. + * + * If no patterns are specified, all classes (that also meet other configured filters) will be included. + */ + SetProperty getExcludeAnnotationClasses(); + + } + } ---- @@ -143,6 +221,34 @@ The plugin supports retrying Spock `@Stepwise` tests and TestNG `@Test(dependsOn * Upstream tests (those that the failed test depends on) are run because a flaky test may depend on state from the prior execution of an upstream test. * Downstream tests are run because a flaky test causes any downstream tests to be skipped in the initial test run. +== Filtering + +By default, all tests are eligible for retrying. +The `filter` component of the test retry extension can be used to control which tests should be retried and which should not. + +The decision to retry a test or not is based on the tests reported class name, regardless of the name of the test case or method. +The annotations present or not on this class can also be used as the criteria. + +.build.gradle: +[source,groovy] +---- +test { + retry { + maxRetries = 3 + filter { + // filter by qualified class name (* matches zero or more of any character) + includeClasses.add("*IntegrationTest") + excludeClasses.add("*DatabaseTest") + + // filter by class level annotations + // Note: @Inherited annotations are respected + includeAnnotationClasses.add("*Retryable") + excludeAnnotationClasses.add("*NonRetryable") + } + } +} +---- + == Reporting === Gradle diff --git a/plugin/build.gradle.kts b/plugin/build.gradle.kts index 0f7c9e9e..17384f74 100644 --- a/plugin/build.gradle.kts +++ b/plugin/build.gradle.kts @@ -40,6 +40,7 @@ dependencies { testImplementation(localGroovy()) testImplementation("org.spockframework:spock-core:1.3-groovy-2.5") testImplementation("net.sourceforge.nekohtml:nekohtml:1.9.22") + testImplementation("org.ow2.asm:asm:8.0.1") codenarc("org.codenarc:CodeNarc:1.0") } diff --git a/plugin/src/main/java/org/gradle/testretry/TestRetryTaskExtension.java b/plugin/src/main/java/org/gradle/testretry/TestRetryTaskExtension.java index a05be7f9..46acdaf6 100644 --- a/plugin/src/main/java/org/gradle/testretry/TestRetryTaskExtension.java +++ b/plugin/src/main/java/org/gradle/testretry/TestRetryTaskExtension.java @@ -15,7 +15,9 @@ */ package org.gradle.testretry; +import org.gradle.api.Action; import org.gradle.api.provider.Property; +import org.gradle.api.provider.SetProperty; import org.gradle.api.tasks.testing.Test; /** @@ -67,4 +69,75 @@ public interface TestRetryTaskExtension { */ Property getMaxFailures(); + /** + * The filter for specifying which tests may be retried. + */ + Filter getFilter(); + + /** + * The filter for specifying which tests may be retried. + */ + void filter(Action action); + + /** + * A filter for specifying which tests may be retried. + * + * By default, all tests are eligible for retrying. + */ + interface Filter { + + /** + * The patterns used to include tests based on their class name. + * + * The pattern string matches against qualified class names. + * It may contain '*' characters, which match zero or more of any character. + * + * A class name only has to match one pattern to be included. + * + * If no patterns are specified, all classes (that also meet other configured filters) will be included. + */ + SetProperty getIncludeClasses(); + + /** + * The patterns used to include tests based on their class level annotations. + * + * The pattern string matches against the qualified class names of a test class's annotations. + * It may contain '*' characters, which match zero or more of any character. + * + * A class need only have one annotation matching any of the patterns to be included. + * + * Annotations present on super classes that are {@code @Inherited} are considered when inspecting subclasses. + * + * If no patterns are specified, all classes (that also meet other configured filters) will be included. + */ + SetProperty getIncludeAnnotationClasses(); + + /** + * The patterns used to exclude tests based on their class name. + * + * The pattern string matches against qualified class names. + * It may contain '*' characters, which match zero or more of any character. + * + * A class name only has to match one pattern to be excluded. + * + * If no patterns are specified, all classes (that also meet other configured filters) will be included. + */ + SetProperty getExcludeClasses(); + + /** + * The patterns used to exclude tests based on their class level annotations. + * + * The pattern string matches against the qualified class names of a test class's annotations. + * It may contain '*' characters, which match zero or more of any character. + * + * A class need only have one annotation matching any of the patterns to be excluded. + * + * Annotations present on super classes that are {@code @Inherited} are considered when inspecting subclasses. + * + * If no patterns are specified, all classes (that also meet other configured filters) will be included. + */ + SetProperty getExcludeAnnotationClasses(); + + } + } diff --git a/plugin/src/main/java/org/gradle/testretry/internal/config/DefaultTestRetryTaskExtension.java b/plugin/src/main/java/org/gradle/testretry/internal/config/DefaultTestRetryTaskExtension.java index 582e3717..3dc0eef8 100644 --- a/plugin/src/main/java/org/gradle/testretry/internal/config/DefaultTestRetryTaskExtension.java +++ b/plugin/src/main/java/org/gradle/testretry/internal/config/DefaultTestRetryTaskExtension.java @@ -15,8 +15,10 @@ */ package org.gradle.testretry.internal.config; +import org.gradle.api.Action; import org.gradle.api.model.ObjectFactory; import org.gradle.api.provider.Property; +import org.gradle.api.provider.SetProperty; import org.gradle.testretry.TestRetryTaskExtension; import javax.inject.Inject; @@ -26,12 +28,14 @@ public class DefaultTestRetryTaskExtension implements TestRetryTaskExtension { private final Property failOnPassedAfterRetry; private final Property maxRetries; private final Property maxFailures; + private final Filter filter; @Inject public DefaultTestRetryTaskExtension(ObjectFactory objects) { this.failOnPassedAfterRetry = objects.property(Boolean.class); this.maxRetries = objects.property(Integer.class); this.maxFailures = objects.property(Integer.class); + this.filter = new FilterImpl(objects); } public Property getFailOnPassedAfterRetry() { @@ -46,4 +50,49 @@ public Property getMaxFailures() { return maxFailures; } + @Override + public void filter(Action action) { + action.execute(filter); + } + + @Override + public Filter getFilter() { + return filter; + } + + private static final class FilterImpl implements Filter { + + private final SetProperty includeClasses; + private final SetProperty includeAnnotationClasses; + private final SetProperty excludeClasses; + private final SetProperty excludeAnnotationClasses; + + public FilterImpl(ObjectFactory objects) { + this.includeClasses = objects.setProperty(String.class); + this.includeAnnotationClasses = objects.setProperty(String.class); + this.excludeClasses = objects.setProperty(String.class); + this.excludeAnnotationClasses = objects.setProperty(String.class); + } + + @Override + public SetProperty getIncludeClasses() { + return includeClasses; + } + + @Override + public SetProperty getIncludeAnnotationClasses() { + return includeAnnotationClasses; + } + + @Override + public SetProperty getExcludeClasses() { + return excludeClasses; + } + + @Override + public SetProperty getExcludeAnnotationClasses() { + return excludeAnnotationClasses; + } + } + } diff --git a/plugin/src/main/java/org/gradle/testretry/internal/config/TestRetryTaskExtensionAdapter.java b/plugin/src/main/java/org/gradle/testretry/internal/config/TestRetryTaskExtensionAdapter.java index cbe246b9..c3976c8a 100644 --- a/plugin/src/main/java/org/gradle/testretry/internal/config/TestRetryTaskExtensionAdapter.java +++ b/plugin/src/main/java/org/gradle/testretry/internal/config/TestRetryTaskExtensionAdapter.java @@ -18,9 +18,12 @@ import org.gradle.api.provider.Property; import org.gradle.api.provider.Provider; import org.gradle.api.provider.ProviderFactory; +import org.gradle.api.provider.SetProperty; import org.gradle.testretry.TestRetryTaskExtension; -import org.jetbrains.annotations.NotNull; +import org.gradle.util.VersionNumber; +import java.util.Collections; +import java.util.Set; import java.util.concurrent.Callable; public final class TestRetryTaskExtensionAdapter { @@ -34,29 +37,44 @@ public final class TestRetryTaskExtensionAdapter { private final ProviderFactory providerFactory; private final TestRetryTaskExtension extension; - private final boolean useConventions; private final boolean simulateNotRetryableTest; + private boolean useConventions; public TestRetryTaskExtensionAdapter( ProviderFactory providerFactory, TestRetryTaskExtension extension, - boolean useConventions + VersionNumber gradleVersion + ) { this.providerFactory = providerFactory; this.extension = extension; - this.useConventions = useConventions; + this.simulateNotRetryableTest = Boolean.getBoolean(SIMULATE_NOT_RETRYABLE_PROPERTY); - simulateNotRetryableTest = Boolean.getBoolean(SIMULATE_NOT_RETRYABLE_PROPERTY); + boolean gradle51OrLater = gradleVersion.getMajor() == 5 + ? gradleVersion.getMinor() >= 1 + : gradleVersion.getMajor() > 5; - if (useConventions) { - setDefaults(extension); - } + this.useConventions = gradle51OrLater; + + initialize(extension, gradle51OrLater); } - private void setDefaults(TestRetryTaskExtension extension) { - extension.getMaxRetries().convention(DEFAULT_MAX_RETRIES); - extension.getMaxFailures().convention(DEFAULT_MAX_FAILURES); - extension.getFailOnPassedAfterRetry().convention(DEFAULT_FAIL_ON_PASSED_AFTER_RETRY); + private void initialize(TestRetryTaskExtension extension, boolean gradle51OrLater) { + if (gradle51OrLater) { + extension.getMaxRetries().convention(DEFAULT_MAX_RETRIES); + extension.getMaxFailures().convention(DEFAULT_MAX_FAILURES); + extension.getFailOnPassedAfterRetry().convention(DEFAULT_FAIL_ON_PASSED_AFTER_RETRY); + extension.getFilter().getIncludeClasses().convention(Collections.emptySet()); + extension.getFilter().getIncludeAnnotationClasses().convention(Collections.emptySet()); + extension.getFilter().getExcludeClasses().convention(Collections.emptySet()); + extension.getFilter().getExcludeAnnotationClasses().convention(Collections.emptySet()); + } else { + // https://github.com/gradle/gradle/issues/7485 + extension.getFilter().getIncludeClasses().empty(); + extension.getFilter().getIncludeAnnotationClasses().empty(); + extension.getFilter().getExcludeClasses().empty(); + extension.getFilter().getExcludeAnnotationClasses().empty(); + } } Callable> getFailOnPassedAfterRetryInput() { @@ -85,12 +103,32 @@ public int getMaxFailures() { return read(extension.getMaxFailures(), DEFAULT_MAX_FAILURES); } + public Set getIncludeClasses() { + return read(extension.getFilter().getIncludeClasses(), Collections.emptySet()); + } + + public Set getIncludeAnnotationClasses() { + return read(extension.getFilter().getIncludeAnnotationClasses(), Collections.emptySet()); + } + + public Set getExcludeClasses() { + return read(extension.getFilter().getExcludeClasses(), Collections.emptySet()); + } + + public Set getExcludeAnnotationClasses() { + return read(extension.getFilter().getExcludeAnnotationClasses(), Collections.emptySet()); + } + public boolean getSimulateNotRetryableTest() { return simulateNotRetryableTest; } - @NotNull private T read(Property property, T defaultValue) { return useConventions ? property.get() : property.getOrElse(defaultValue); } + + private Set read(SetProperty property, Set defaultValue) { + return useConventions ? property.get() : property.getOrElse(defaultValue); + } + } diff --git a/plugin/src/main/java/org/gradle/testretry/internal/config/TestTaskConfigurer.java b/plugin/src/main/java/org/gradle/testretry/internal/config/TestTaskConfigurer.java index ed895398..81cc5b02 100644 --- a/plugin/src/main/java/org/gradle/testretry/internal/config/TestTaskConfigurer.java +++ b/plugin/src/main/java/org/gradle/testretry/internal/config/TestTaskConfigurer.java @@ -40,14 +40,9 @@ private TestTaskConfigurer() { public static void configureTestTask(Test test, ObjectFactory objectFactory, ProviderFactory providerFactory) { VersionNumber gradleVersion = VersionNumber.parse(test.getProject().getGradle().getGradleVersion()); - TestRetryTaskExtension extension; - if (supportsGeneratedAbstractTypeImplementations(gradleVersion)) { - extension = objectFactory.newInstance(TestRetryTaskExtension.class); - } else { - extension = objectFactory.newInstance(DefaultTestRetryTaskExtension.class); - } + TestRetryTaskExtension extension = objectFactory.newInstance(DefaultTestRetryTaskExtension.class); - TestRetryTaskExtensionAdapter adapter = new TestRetryTaskExtensionAdapter(providerFactory, extension, supportsPropertyConventions(gradleVersion)); + TestRetryTaskExtensionAdapter adapter = new TestRetryTaskExtensionAdapter(providerFactory, extension, gradleVersion); test.getInputs().property("retry.failOnPassedAfterRetry", adapter.getFailOnPassedAfterRetryInput()); @@ -70,10 +65,6 @@ private static void setTestExecuter(Test task, RetryTestExecuter retryTestExecut invoke(declaredMethod(Test.class, "setTestExecuter", TestExecuter.class), task, retryTestExecuter); } - private static boolean supportsGeneratedAbstractTypeImplementations(VersionNumber gradleVersion) { - return gradleVersion.getMajor() == 5 ? gradleVersion.getMinor() >= 3 : gradleVersion.getMajor() > 5; - } - private static boolean supportsPropertyConventions(VersionNumber gradleVersion) { return gradleVersion.getMajor() == 5 ? gradleVersion.getMinor() >= 1 : gradleVersion.getMajor() > 5; } diff --git a/plugin/src/main/java/org/gradle/testretry/internal/executer/RetryTestExecuter.java b/plugin/src/main/java/org/gradle/testretry/internal/executer/RetryTestExecuter.java index f37888cc..36244039 100644 --- a/plugin/src/main/java/org/gradle/testretry/internal/executer/RetryTestExecuter.java +++ b/plugin/src/main/java/org/gradle/testretry/internal/executer/RetryTestExecuter.java @@ -24,6 +24,8 @@ import org.gradle.internal.reflect.Instantiator; import org.gradle.testretry.internal.config.TestRetryTaskExtensionAdapter; import org.gradle.testretry.internal.executer.framework.TestFrameworkStrategy; +import org.gradle.testretry.internal.filter.AnnotationInspectorImpl; +import org.gradle.testretry.internal.filter.RetryFilter; import java.util.stream.Collectors; @@ -66,8 +68,17 @@ public void execute(JvmTestExecutionSpec spec, TestResultProcessor testResultPro TestFrameworkStrategy testFrameworkStrategy = TestFrameworkStrategy.of(spec.getTestFramework()); + RetryFilter filter = new RetryFilter( + new AnnotationInspectorImpl(frameworkTemplate.testsReader), + extension.getIncludeClasses(), + extension.getIncludeAnnotationClasses(), + extension.getExcludeClasses(), + extension.getExcludeAnnotationClasses() + ); + RetryTestResultProcessor retryTestResultProcessor = new RetryTestResultProcessor( testFrameworkStrategy, + filter, frameworkTemplate.testsReader, testResultProcessor, maxFailures @@ -86,7 +97,7 @@ public void execute(JvmTestExecutionSpec spec, TestResultProcessor testResultPro testTask.setIgnoreFailures(true); break; } else if (result.failedTests.isEmpty()) { - if (retryCount > 0 && !failOnPassedAfterRetry) { + if (retryCount > 0 && !result.hasRetryFilteredFailures && !failOnPassedAfterRetry) { testTask.setIgnoreFailures(true); } break; diff --git a/plugin/src/main/java/org/gradle/testretry/internal/executer/RetryTestResultProcessor.java b/plugin/src/main/java/org/gradle/testretry/internal/executer/RetryTestResultProcessor.java index c6829cb3..6170b555 100644 --- a/plugin/src/main/java/org/gradle/testretry/internal/executer/RetryTestResultProcessor.java +++ b/plugin/src/main/java/org/gradle/testretry/internal/executer/RetryTestResultProcessor.java @@ -21,6 +21,8 @@ import org.gradle.api.internal.tasks.testing.TestStartEvent; import org.gradle.api.tasks.testing.TestOutputEvent; import org.gradle.testretry.internal.executer.framework.TestFrameworkStrategy; +import org.gradle.testretry.internal.filter.RetryFilter; +import org.gradle.testretry.internal.testsreader.TestsReader; import java.util.HashMap; import java.util.Map; @@ -30,11 +32,13 @@ final class RetryTestResultProcessor implements TestResultProcessor { private final TestFrameworkStrategy testFrameworkStrategy; + private final RetryFilter filter; private final TestsReader testsReader; private final TestResultProcessor delegate; private final int maxFailures; private boolean lastRetry; + private boolean hasRetryFilteredFailures; private final Map activeDescriptorsById = new HashMap<>(); @@ -45,11 +49,13 @@ final class RetryTestResultProcessor implements TestResultProcessor { RetryTestResultProcessor( TestFrameworkStrategy testFrameworkStrategy, + RetryFilter filter, TestsReader testsReader, TestResultProcessor delegate, int maxFailures ) { this.testFrameworkStrategy = testFrameworkStrategy; + this.filter = filter; this.testsReader = testsReader; this.delegate = delegate; this.maxFailures = maxFailures; @@ -121,8 +127,15 @@ public void output(Object testId, TestOutputEvent testOutputEvent) { @Override public void failure(Object testId, Throwable throwable) { final TestDescriptorInternal descriptor = activeDescriptorsById.get(testId); - if (descriptor != null && descriptor.getClassName() != null) { - currentRoundFailedTests.add(descriptor.getClassName(), descriptor.getName()); + if (descriptor != null) { + String className = descriptor.getClassName(); + if (className != null) { + if (filter.canRetry(className)) { + currentRoundFailedTests.add(className, descriptor.getName()); + } else { + hasRetryFilteredFailures = true; + } + } } delegate.failure(testId, throwable); @@ -135,7 +148,7 @@ private boolean lastRun() { } public RoundResult getResult() { - return new RoundResult(currentRoundFailedTests, previousRoundFailedTests, lastRun()); + return new RoundResult(currentRoundFailedTests, previousRoundFailedTests, lastRun(), hasRetryFilteredFailures); } public void reset(boolean lastRetry) { diff --git a/plugin/src/main/java/org/gradle/testretry/internal/executer/RoundResult.java b/plugin/src/main/java/org/gradle/testretry/internal/executer/RoundResult.java index 9ab79add..5beb5e06 100644 --- a/plugin/src/main/java/org/gradle/testretry/internal/executer/RoundResult.java +++ b/plugin/src/main/java/org/gradle/testretry/internal/executer/RoundResult.java @@ -20,10 +20,17 @@ final class RoundResult { final TestNames failedTests; final TestNames nonRetriedTests; final boolean lastRound; + final boolean hasRetryFilteredFailures; - RoundResult(TestNames failedTests, TestNames nonRetriedTests, boolean lastRound) { + RoundResult( + TestNames failedTests, + TestNames nonRetriedTests, + boolean lastRound, + boolean hasRetryFilteredFailures + ) { this.failedTests = failedTests; this.nonRetriedTests = nonRetriedTests; this.lastRound = lastRound; + this.hasRetryFilteredFailures = hasRetryFilteredFailures; } } diff --git a/plugin/src/main/java/org/gradle/testretry/internal/executer/TestFrameworkTemplate.java b/plugin/src/main/java/org/gradle/testretry/internal/executer/TestFrameworkTemplate.java index b6631d6b..00c7fa8b 100644 --- a/plugin/src/main/java/org/gradle/testretry/internal/executer/TestFrameworkTemplate.java +++ b/plugin/src/main/java/org/gradle/testretry/internal/executer/TestFrameworkTemplate.java @@ -18,6 +18,7 @@ import org.gradle.api.model.ObjectFactory; import org.gradle.api.tasks.testing.Test; import org.gradle.internal.reflect.Instantiator; +import org.gradle.testretry.internal.testsreader.TestsReader; public class TestFrameworkTemplate { diff --git a/plugin/src/main/java/org/gradle/testretry/internal/executer/framework/BaseJunitTestFrameworkStrategy.java b/plugin/src/main/java/org/gradle/testretry/internal/executer/framework/BaseJunitTestFrameworkStrategy.java index e1cb2209..9354081e 100644 --- a/plugin/src/main/java/org/gradle/testretry/internal/executer/framework/BaseJunitTestFrameworkStrategy.java +++ b/plugin/src/main/java/org/gradle/testretry/internal/executer/framework/BaseJunitTestFrameworkStrategy.java @@ -17,7 +17,7 @@ import org.gradle.testretry.internal.executer.TestFilterBuilder; import org.gradle.testretry.internal.executer.TestNames; -import org.gradle.testretry.internal.executer.TestsReader; +import org.gradle.testretry.internal.testsreader.TestsReader; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/plugin/src/main/java/org/gradle/testretry/internal/executer/framework/SpockParameterClassVisitor.java b/plugin/src/main/java/org/gradle/testretry/internal/executer/framework/SpockParameterClassVisitor.java index db185364..e6c82cfc 100644 --- a/plugin/src/main/java/org/gradle/testretry/internal/executer/framework/SpockParameterClassVisitor.java +++ b/plugin/src/main/java/org/gradle/testretry/internal/executer/framework/SpockParameterClassVisitor.java @@ -15,7 +15,7 @@ */ package org.gradle.testretry.internal.executer.framework; -import org.gradle.testretry.internal.executer.TestsReader; +import org.gradle.testretry.internal.testsreader.TestsReader; import org.objectweb.asm.AnnotationVisitor; import org.objectweb.asm.MethodVisitor; diff --git a/plugin/src/main/java/org/gradle/testretry/internal/executer/framework/SpockStepwiseClassVisitor.java b/plugin/src/main/java/org/gradle/testretry/internal/executer/framework/SpockStepwiseClassVisitor.java index b37297fa..7b4029fc 100644 --- a/plugin/src/main/java/org/gradle/testretry/internal/executer/framework/SpockStepwiseClassVisitor.java +++ b/plugin/src/main/java/org/gradle/testretry/internal/executer/framework/SpockStepwiseClassVisitor.java @@ -15,7 +15,7 @@ */ package org.gradle.testretry.internal.executer.framework; -import org.gradle.testretry.internal.executer.TestsReader; +import org.gradle.testretry.internal.testsreader.TestsReader; import org.objectweb.asm.AnnotationVisitor; final class SpockStepwiseClassVisitor extends TestsReader.Visitor { diff --git a/plugin/src/main/java/org/gradle/testretry/internal/executer/framework/TestFrameworkStrategy.java b/plugin/src/main/java/org/gradle/testretry/internal/executer/framework/TestFrameworkStrategy.java index fdefe070..1d5b024b 100644 --- a/plugin/src/main/java/org/gradle/testretry/internal/executer/framework/TestFrameworkStrategy.java +++ b/plugin/src/main/java/org/gradle/testretry/internal/executer/framework/TestFrameworkStrategy.java @@ -21,7 +21,7 @@ import org.gradle.api.internal.tasks.testing.testng.TestNGTestFramework; import org.gradle.testretry.internal.executer.TestFrameworkTemplate; import org.gradle.testretry.internal.executer.TestNames; -import org.gradle.testretry.internal.executer.TestsReader; +import org.gradle.testretry.internal.testsreader.TestsReader; import org.gradle.util.GradleVersion; /** diff --git a/plugin/src/main/java/org/gradle/testretry/internal/executer/framework/TestNgClassVisitor.java b/plugin/src/main/java/org/gradle/testretry/internal/executer/framework/TestNgClassVisitor.java index ae7a0fb3..a86736ef 100644 --- a/plugin/src/main/java/org/gradle/testretry/internal/executer/framework/TestNgClassVisitor.java +++ b/plugin/src/main/java/org/gradle/testretry/internal/executer/framework/TestNgClassVisitor.java @@ -15,7 +15,7 @@ */ package org.gradle.testretry.internal.executer.framework; -import org.gradle.testretry.internal.executer.TestsReader; +import org.gradle.testretry.internal.testsreader.TestsReader; import org.objectweb.asm.AnnotationVisitor; import org.objectweb.asm.MethodVisitor; diff --git a/plugin/src/main/java/org/gradle/testretry/internal/executer/framework/TestNgTestFrameworkStrategy.java b/plugin/src/main/java/org/gradle/testretry/internal/executer/framework/TestNgTestFrameworkStrategy.java index deffca5e..0dd2e82c 100644 --- a/plugin/src/main/java/org/gradle/testretry/internal/executer/framework/TestNgTestFrameworkStrategy.java +++ b/plugin/src/main/java/org/gradle/testretry/internal/executer/framework/TestNgTestFrameworkStrategy.java @@ -27,7 +27,7 @@ import org.gradle.testretry.internal.executer.TestFilterBuilder; import org.gradle.testretry.internal.executer.TestFrameworkTemplate; import org.gradle.testretry.internal.executer.TestNames; -import org.gradle.testretry.internal.executer.TestsReader; +import org.gradle.testretry.internal.testsreader.TestsReader; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/plugin/src/main/java/org/gradle/testretry/internal/filter/AnnotationInspector.java b/plugin/src/main/java/org/gradle/testretry/internal/filter/AnnotationInspector.java new file mode 100644 index 00000000..0ec94f08 --- /dev/null +++ b/plugin/src/main/java/org/gradle/testretry/internal/filter/AnnotationInspector.java @@ -0,0 +1,24 @@ +/* + * Copyright 2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.gradle.testretry.internal.filter; + +import java.util.Set; + +public interface AnnotationInspector { + + Set getClassAnnotations(String className); + +} diff --git a/plugin/src/main/java/org/gradle/testretry/internal/filter/AnnotationInspectorImpl.java b/plugin/src/main/java/org/gradle/testretry/internal/filter/AnnotationInspectorImpl.java new file mode 100644 index 00000000..adc52db7 --- /dev/null +++ b/plugin/src/main/java/org/gradle/testretry/internal/filter/AnnotationInspectorImpl.java @@ -0,0 +1,114 @@ +/* + * Copyright 2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.gradle.testretry.internal.filter; + +import org.gradle.api.logging.Logger; +import org.gradle.api.logging.Logging; +import org.gradle.testretry.internal.testsreader.TestsReader; +import org.objectweb.asm.AnnotationVisitor; + +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +public class AnnotationInspectorImpl implements AnnotationInspector { + + private static final Logger LOGGER = Logging.getLogger(AnnotationInspectorImpl.class); + + private final Map> cache = new HashMap<>(); + private final Map inheritedCache = new HashMap<>(); + + private final TestsReader testsReader; + + public AnnotationInspectorImpl(TestsReader testsReader) { + this.testsReader = testsReader; + } + + @Override + public Set getClassAnnotations(String className) { + Set annotations = cache.get(className); + if (annotations == null) { + annotations = testsReader.readClass(className, ClassAnnotationVisitor::new) + .orElseGet(() -> { + LOGGER.warn("Unable to find annotations of " + className); + return Collections.emptySet(); + }); + cache.put(className, annotations); + } + return annotations; + } + + private boolean isInherited(String annotationClassName) { + return inheritedCache.computeIfAbsent(annotationClassName, ignored -> + testsReader.readClass(annotationClassName, AnnotationAnnotationVisitor::new) + .orElseGet(() -> { + LOGGER.warn("Cannot determine whether @" + annotationClassName + " is inherited"); + return false; + }) + ); + } + + final class ClassAnnotationVisitor extends TestsReader.Visitor> { + + private final Set found = new HashSet<>(); + + @Override + public Set getResult() { + return found.isEmpty() ? Collections.emptySet() : found; + } + + @Override + public void visit(int version, int access, String name, String signature, String superName, String[] interfaces) { + if (!superName.equals("java/lang/Object")) { + getClassAnnotations(superName.replace('/', '.')) + .stream() + .filter(AnnotationInspectorImpl.this::isInherited) + .forEach(found::add); + } + } + + @Override + public AnnotationVisitor visitAnnotation(String descriptor, boolean visible) { + found.add(classDescriptorToClassName(descriptor)); + return null; + } + + } + + static final class AnnotationAnnotationVisitor extends TestsReader.Visitor { + + private boolean inherited; + + @Override + public Boolean getResult() { + return inherited; + } + + @Override + public AnnotationVisitor visitAnnotation(String descriptor, boolean visible) { + if (descriptor.equals("Ljava/lang/annotation/Inherited;")) { + inherited = true; + } + return null; + } + } + + private static String classDescriptorToClassName(String descriptor) { + return descriptor.substring(1, descriptor.length() - 1).replace('/', '.'); + } +} diff --git a/plugin/src/main/java/org/gradle/testretry/internal/filter/GlobPattern.java b/plugin/src/main/java/org/gradle/testretry/internal/filter/GlobPattern.java new file mode 100644 index 00000000..72d6887e --- /dev/null +++ b/plugin/src/main/java/org/gradle/testretry/internal/filter/GlobPattern.java @@ -0,0 +1,75 @@ +/* + * Copyright 2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.gradle.testretry.internal.filter; + +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +final class GlobPattern { + + public static final Pattern STAR_CHAR_PATTERN = Pattern.compile("\\*"); + public static final String MATCH_ANY = ".*?"; + + private final String string; + private final Pattern pattern; + + private GlobPattern(String string, Pattern pattern) { + this.string = string; + this.pattern = pattern; + } + + static GlobPattern from(String string) { + String patternString = STAR_CHAR_PATTERN.splitAsStream(string) + .map(Pattern::quote) + .collect(Collectors.joining(MATCH_ANY)); + + if (string.endsWith("*")) { + patternString = patternString + MATCH_ANY; + } + + Pattern pattern = Pattern.compile(patternString); + + return new GlobPattern(patternString, pattern); + } + + boolean matches(String test) { + return pattern.matcher(test).matches(); + } + + @Override + public String toString() { + return string; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + GlobPattern that = (GlobPattern) o; + + return string.equals(that.string); + } + + @Override + public int hashCode() { + return string.hashCode(); + } +} diff --git a/plugin/src/main/java/org/gradle/testretry/internal/filter/RetryFilter.java b/plugin/src/main/java/org/gradle/testretry/internal/filter/RetryFilter.java new file mode 100644 index 00000000..5372b4cd --- /dev/null +++ b/plugin/src/main/java/org/gradle/testretry/internal/filter/RetryFilter.java @@ -0,0 +1,84 @@ +/* + * Copyright 2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.gradle.testretry.internal.filter; + +import java.util.Collection; +import java.util.Collections; +import java.util.Set; +import java.util.stream.Collectors; + +public class RetryFilter { + + private final AnnotationInspector annotationInspector; + + private final Set includeClasses; + private final Set includeAnnotationClasses; + private final Set excludeClasses; + private final Set excludeAnnotationClasses; + + public RetryFilter( + AnnotationInspector annotationInspector, + Collection includeClasses, + Collection includeAnnotationClasses, + Collection excludeClasses, + Collection excludeAnnotationClasses + ) { + this.annotationInspector = annotationInspector; + this.includeClasses = toPatterns(includeClasses); + this.includeAnnotationClasses = toPatterns(includeAnnotationClasses); + this.excludeClasses = toPatterns(excludeClasses); + this.excludeAnnotationClasses = toPatterns(excludeAnnotationClasses); + } + + public boolean canRetry(String className) { + if (!includeClasses.isEmpty()) { + if (!anyMatch(includeClasses, className)) { + return false; + } + } + + if (anyMatch(excludeClasses, className)) { + return false; + } + + Set annotations = null; // fetching annotations is expensive, don't do it unnecessarily. + if (!includeAnnotationClasses.isEmpty()) { + annotations = annotationInspector.getClassAnnotations(className); + if (annotations.isEmpty() || !anyMatch(includeAnnotationClasses, annotations)) { + return false; + } + } + + if (!excludeAnnotationClasses.isEmpty()) { + annotations = annotations == null ? annotationInspector.getClassAnnotations(className) : annotations; + return !anyMatch(excludeAnnotationClasses, annotations); + } + + return true; + } + + private static boolean anyMatch(Set patterns, String string) { + return anyMatch(patterns, Collections.singleton(string)); + } + + private static boolean anyMatch(Set patterns, Set strings) { + return patterns.stream().anyMatch(p -> strings.stream().anyMatch(p::matches)); + } + + private static Set toPatterns(Collection strings) { + return strings.stream().map(GlobPattern::from).collect(Collectors.toSet()); + } +} diff --git a/plugin/src/main/java/org/gradle/testretry/internal/executer/TestsReader.java b/plugin/src/main/java/org/gradle/testretry/internal/testsreader/TestsReader.java similarity index 98% rename from plugin/src/main/java/org/gradle/testretry/internal/executer/TestsReader.java rename to plugin/src/main/java/org/gradle/testretry/internal/testsreader/TestsReader.java index a13c1746..f824fbdb 100644 --- a/plugin/src/main/java/org/gradle/testretry/internal/executer/TestsReader.java +++ b/plugin/src/main/java/org/gradle/testretry/internal/testsreader/TestsReader.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.gradle.testretry.internal.executer; +package org.gradle.testretry.internal.testsreader; import org.jetbrains.annotations.NotNull; import org.objectweb.asm.ClassReader; diff --git a/plugin/src/test/groovy/org/gradle/testretry/AbstractGeneralPluginFuncTest.groovy b/plugin/src/test/groovy/org/gradle/testretry/AbstractGeneralPluginFuncTest.groovy index 52633ddb..dcb8dabd 100644 --- a/plugin/src/test/groovy/org/gradle/testretry/AbstractGeneralPluginFuncTest.groovy +++ b/plugin/src/test/groovy/org/gradle/testretry/AbstractGeneralPluginFuncTest.groovy @@ -17,10 +17,6 @@ package org.gradle.testretry abstract class AbstractGeneralPluginFuncTest extends AbstractPluginFuncTest { - String getTestAnnotation() { - return '@org.junit.Test' - } - String getLanguagePlugin() { return 'java' } @@ -30,7 +26,7 @@ abstract class AbstractGeneralPluginFuncTest extends AbstractPluginFuncTest { package acme; public class SuccessfulTests { - ${testAnnotation} + @org.junit.Test public void successTest() {} } """ @@ -56,7 +52,7 @@ abstract class AbstractGeneralPluginFuncTest extends AbstractPluginFuncTest { package acme; public class FlakyTests { - ${testAnnotation} + @org.junit.Test public void flaky() { ${flakyAssert()} } diff --git a/plugin/src/test/groovy/org/gradle/testretry/FilterFuncTest.groovy b/plugin/src/test/groovy/org/gradle/testretry/FilterFuncTest.groovy new file mode 100644 index 00000000..b2be7f4e --- /dev/null +++ b/plugin/src/test/groovy/org/gradle/testretry/FilterFuncTest.groovy @@ -0,0 +1,115 @@ +/* + * Copyright 2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.gradle.testretry + +import spock.lang.Unroll + +import javax.annotation.Nullable + +class FilterFuncTest extends AbstractGeneralPluginFuncTest { + + @Unroll + def "can filter what is retried (gradle version #gradleVersion)"() { + given: + buildFile << """ + test.retry { + maxRetries = 2 + filter { + includeClasses.add("*Included*") + includeAnnotationClasses.add("*Included*") + excludeClasses.add("*Excluded*") + excludeAnnotationClasses.add("*Excluded*") + } + } + """ + + and: + inheritedAnnotation("InheritedIncludedAnnotation") + inheritedAnnotation("InheritedExcludedAnnotation") + nonInheritedAnnotation("NonInheritedIncludedAnnotation") + nonInheritedAnnotation("NonInheritedExcludedAnnotation") + + and: + def noRetry = [] + def shouldRetry = [] + + shouldRetry << test("BaseIncludedTest", null, "InheritedIncludedAnnotation") + noRetry << test("BaseIncludedAndExcludedTest", "BaseIncludedTest", "InheritedExcludedAnnotation") + noRetry << test("IncludedAndExcludedInheritedTest", "BaseIncludedAndExcludedTest") + shouldRetry << test("IncludedNonInheritedTest", null, "NonInheritedIncludedAnnotation") + noRetry << test("ExcludedTest", null, "NonInheritedIncludedAnnotation") + noRetry << test("IncludeWithBadAnnotation", null, "InheritedExcludedAnnotation") + + when: + def result = gradleRunner(gradleVersion).buildAndFail() + + then: + noRetry.each { + assert result.output.count("acme.${it} > flakyTest FAILED") == 1 + assert result.output.count("acme.${it} > flakyTest PASSED") == 0 + } + shouldRetry.each { + assert result.output.count("acme.${it} > flakyTest FAILED") == 2 + assert result.output.count("acme.${it} > flakyTest PASSED") == 1 + } + + where: + gradleVersion << GRADLE_VERSIONS_UNDER_TEST + } + + private void nonInheritedAnnotation(String name) { + file("src/test/java/acme/${name}.java") << """ + package acme; + import java.lang.annotation.ElementType; + import java.lang.annotation.Retention; + import java.lang.annotation.RetentionPolicy; + import java.lang.annotation.Target; + + @Retention(RetentionPolicy.RUNTIME) + @Target({ ElementType.TYPE }) + public @interface $name { } + """ + } + + private void inheritedAnnotation(String name) { + file("src/test/java/${name}.java") << """ + package acme; + import java.lang.annotation.ElementType; + import java.lang.annotation.Retention; + import java.lang.annotation.RetentionPolicy; + import java.lang.annotation.Target; + + @Retention(RetentionPolicy.RUNTIME) + @Target({ ElementType.TYPE }) + @java.lang.annotation.Inherited + public @interface $name { } + """ + } + + private String test(String name, @Nullable String superClass, String... annotations) { + file("src/test/java/acme/${name}.java") << """ + package acme; + ${annotations.collect { "@$it" }.join("\n")} + public class $name ${superClass ? " extends $superClass" : ""} { + @org.junit.Test + public void flakyTest() { + ${flakyAssert(name, 2)} + } + } + """ + return name + } +} diff --git a/plugin/src/test/groovy/org/gradle/testretry/internal/filter/AnnotationInspectorImplTest.groovy b/plugin/src/test/groovy/org/gradle/testretry/internal/filter/AnnotationInspectorImplTest.groovy new file mode 100644 index 00000000..8170e499 --- /dev/null +++ b/plugin/src/test/groovy/org/gradle/testretry/internal/filter/AnnotationInspectorImplTest.groovy @@ -0,0 +1,119 @@ +/* + * Copyright 2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.gradle.testretry.internal.filter + +import org.gradle.testkit.runner.GradleRunner +import org.gradle.testretry.internal.testsreader.TestsReader +import org.junit.Rule +import org.junit.rules.TemporaryFolder +import spock.lang.Specification + +import javax.annotation.Nullable + +class AnnotationInspectorImplTest extends Specification { + + @Rule + TemporaryFolder dir = new TemporaryFolder() + + AnnotationInspector inspector + + def setup() { + def settingsFile = dir.newFile('settings.gradle') + settingsFile << "rootProject.name = 'hello-world'" + def buildFile = dir.newFile('build.gradle') + buildFile << "plugins { id 'java' }" + } + + def "finds annotations"() { + given: + nonInheritedAnnotation("AN1") + nonInheritedAnnotation("AN2") + nonInheritedAnnotation("AN3") + inheritedAnnotation("AI1") + inheritedAnnotation("AI2") + inheritedAnnotation("AI3") + + classWithAnnotations("NoAnnotationBase", null) + classWithAnnotations("IncludeAnnotationBase", null, "AN1", "AI1") + classWithAnnotations("IncludeAnnotationChild", "IncludeAnnotationBase", "AN2", "AI2") + classWithAnnotations("IncludeAnnotationChildChild", "IncludeAnnotationChild") + classWithAnnotations("IncludeAnnotationChildChildChild", "IncludeAnnotationChildChild") + + expect: + annotationsOf("NoAnnotationBase").empty + annotationsOf("NotExist").empty + annotationsOf("IncludeAnnotationBase") == ["AI1", "AN1"] + annotationsOf("IncludeAnnotationChild") == ["AI1", "AI2", "AN2"] + annotationsOf("IncludeAnnotationChildChild") == ["AI1", "AI2"] + annotationsOf("IncludeAnnotationChildChildChild") == ["AI1", "AI2"] + } + + List annotationsOf(String className) { + if (inspector == null) { + inspector = inspector() + } + inspector.getClassAnnotations(className).toList().sort() + } + + File file(String path) { + def file = new File(dir.root, path) + assert file.parentFile.mkdirs() || file.parentFile.directory + assert file.createNewFile() || file.file + file + } + + AnnotationInspector inspector() { + GradleRunner.create().withProjectDir(dir.root).withArguments("compileJava").build() + def reader = new TestsReader([new File(dir.root, "build/classes/java/main")].toSet(), []) + new AnnotationInspectorImpl(reader) + } + + private void nonInheritedAnnotation(String name) { + file("src/main/java/${name}.java") << """ + import java.lang.annotation.ElementType; + import java.lang.annotation.Retention; + import java.lang.annotation.RetentionPolicy; + import java.lang.annotation.Target; + + @Retention(RetentionPolicy.RUNTIME) + @Target({ ElementType.TYPE }) + public @interface $name { } + """ + } + + private void inheritedAnnotation(String name) { + file("src/main/java/${name}.java") << """ + import java.lang.annotation.ElementType; + import java.lang.annotation.Retention; + import java.lang.annotation.RetentionPolicy; + import java.lang.annotation.Target; + + @Retention(RetentionPolicy.RUNTIME) + @Target({ ElementType.TYPE }) + @java.lang.annotation.Inherited + public @interface $name { } + """ + } + + private void classWithAnnotations(String name, @Nullable String superClass, String... annotations) { + file("src/main/java/${name}.java") << """ + ${annotations.collect { "@$it" }.join("\n")} + class $name ${superClass ? " extends $superClass" : ""} {} + """ + + } + +} diff --git a/plugin/src/test/groovy/org/gradle/testretry/internal/filter/GlobPatternTest.groovy b/plugin/src/test/groovy/org/gradle/testretry/internal/filter/GlobPatternTest.groovy new file mode 100644 index 00000000..1032870f --- /dev/null +++ b/plugin/src/test/groovy/org/gradle/testretry/internal/filter/GlobPatternTest.groovy @@ -0,0 +1,59 @@ +/* + * Copyright 2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.gradle.testretry.internal.filter + +import spock.lang.Specification + +class GlobPatternTest extends Specification { + + def "glob pattern matching"() { + expect: + with(GlobPattern.from("*")) { + matches "a" + matches "b" + matches "*" + matches " " + } + with(GlobPattern.from("**")) { + matches "a" + matches "b" + matches "*" + matches " " + } + with(GlobPattern.from(".")) { + matches "." + !matches("b") + } + with(GlobPattern.from("a*")) { + matches "a" + matches "ab" + !matches("ba") + !matches("b") + } + with(GlobPattern.from("a*a")) { + matches "aba" + matches "aa" + !matches("ba") + !matches("abc") + } + with(GlobPattern.from("**")) { + matches "aba" + matches "aa" + matches "" + matches "a" + } + } +} diff --git a/plugin/src/test/groovy/org/gradle/testretry/internal/filter/RetryFilterTest.groovy b/plugin/src/test/groovy/org/gradle/testretry/internal/filter/RetryFilterTest.groovy new file mode 100644 index 00000000..a235fc80 --- /dev/null +++ b/plugin/src/test/groovy/org/gradle/testretry/internal/filter/RetryFilterTest.groovy @@ -0,0 +1,107 @@ +/* + * Copyright 2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.gradle.testretry.internal.filter + +import spock.lang.Specification + +class RetryFilterTest extends Specification { + + List includeClasses = [] + List excludeClasses = [] + List includeAnnotations = [] + List excludeAnnotations = [] + + Map> annotations = [:] + + def "empty filter allows all"() { + expect: + with(filter()) { + canRetry("foo.bar") + canRetry("foo.baz") + } + } + + def "must match include pattern"() { + when: + includeClasses << "1.*" << "2.*" + + then: + with(filter()) { + canRetry("1.Test") + canRetry("2.Test") + !canRetry("3.Test") + } + } + + def "must not match exclude pattern"() { + when: + includeClasses << "1.*" << "2.*" + excludeClasses << "2.*" << "3.*" << "z" + + then: + with(filter()) { + canRetry("1.Test") + !canRetry("2.Test") + !canRetry("3.Test") + !canRetry("2.z") + } + } + + def "must have include annotation"() { + when: + includeClasses << "*include*" + excludeClasses << "*exclude*" + includeAnnotations << "*include*" + annotations["include1"] = ["a"] + annotations["include2"] = ["a", "include"] + + then: + with(filter()) { + !canRetry("include1") + canRetry("include2") + !canRetry("include3") + } + } + + def "must not have exclude annotation"() { + when: + includeClasses << "*include*" + excludeClasses << "*exclude*" + includeAnnotations << "*include*" + excludeAnnotations << "*exclude*" + annotations["include1"] = ["a"] + annotations["include2"] = ["a", "include", "exclude"] + annotations["include3"] = ["a", "include"] + + then: + with(filter()) { + !canRetry("include1") + !canRetry("include2") + canRetry("include3") + !canRetry("include4") + } + } + + RetryFilter filter() { + new RetryFilter( + { annotations.getOrDefault(it, []).toSet() }, + includeClasses, + includeAnnotations, + excludeClasses, + excludeAnnotations + ) + } +} diff --git a/plugin/src/test/groovy/org/gradle/testretry/testframework/JUnit4FuncTest.groovy b/plugin/src/test/groovy/org/gradle/testretry/testframework/JUnit4FuncTest.groovy index 218ccf2c..1228b40c 100644 --- a/plugin/src/test/groovy/org/gradle/testretry/testframework/JUnit4FuncTest.groovy +++ b/plugin/src/test/groovy/org/gradle/testretry/testframework/JUnit4FuncTest.groovy @@ -26,11 +26,6 @@ class JUnit4FuncTest extends AbstractFrameworkFuncTest { return 'java' } - @Override - String getTestAnnotation() { - return "@org.junit.Test" - } - protected isRerunsAllParameterizedIterations() { false } diff --git a/plugin/src/test/groovy/org/gradle/testretry/testframework/JUnit5FuncTest.groovy b/plugin/src/test/groovy/org/gradle/testretry/testframework/JUnit5FuncTest.groovy index 8171682a..39d18b05 100644 --- a/plugin/src/test/groovy/org/gradle/testretry/testframework/JUnit5FuncTest.groovy +++ b/plugin/src/test/groovy/org/gradle/testretry/testframework/JUnit5FuncTest.groovy @@ -25,11 +25,6 @@ class JUnit5FuncTest extends AbstractFrameworkFuncTest { return 'java' } - @Override - String getTestAnnotation() { - return "@org.junit.jupiter.api.Test" - } - protected String afterClassErrorTestMethodName(String gradleVersion) { gradleVersion == "5.0" ? "classMethod" : "executionError" } diff --git a/plugin/src/test/groovy/org/gradle/testretry/testframework/SpockFuncTest.groovy b/plugin/src/test/groovy/org/gradle/testretry/testframework/SpockFuncTest.groovy index 7a1f325c..3fbfcc14 100644 --- a/plugin/src/test/groovy/org/gradle/testretry/testframework/SpockFuncTest.groovy +++ b/plugin/src/test/groovy/org/gradle/testretry/testframework/SpockFuncTest.groovy @@ -27,11 +27,6 @@ class SpockFuncTest extends AbstractFrameworkFuncTest { return 'groovy' } - @Override - String getTestAnnotation() { - return '' - } - boolean isRerunsParameterizedMethods() { true } diff --git a/plugin/src/test/groovy/org/gradle/testretry/testframework/TestNGFuncTest.groovy b/plugin/src/test/groovy/org/gradle/testretry/testframework/TestNGFuncTest.groovy index fcfbb416..df2abd4b 100644 --- a/plugin/src/test/groovy/org/gradle/testretry/testframework/TestNGFuncTest.groovy +++ b/plugin/src/test/groovy/org/gradle/testretry/testframework/TestNGFuncTest.groovy @@ -25,11 +25,6 @@ class TestNGFuncTest extends AbstractFrameworkFuncTest { return 'java' } - @Override - String getTestAnnotation() { - return "@org.testng.annotations.Test" - } - @Unroll def "handles failure in #lifecycle (gradle version #gradleVersion)"() { given: