Bug 545409 - Add JUnit Rule to retry flaky tests

Add MultiTryTestRule which allows to allow multiple tries for unit
tests. This can either be done for all tests of a test class or be
limited to tests annotated with the new MultiTry annotation.

Change-Id: I01a86edb64d6693f78c33c5751e8673e748c4c68
Signed-off-by: Lucas Koehler <lkoehler@eclipsesource.com>

2 files changed, 127 insertions, 0 deletions

diff --git a/bundles/org.eclipse.emf.ecp.test.common/src/org/eclipse/emf/ecp/test/common/MultiTry.java b/bundles/org.eclipse.emf.ecp.test.common/src/org/eclipse/emf/ecp/test/common/MultiTry.java
new file mode 100644
index 0000000000..44b6b5e244
--- /dev/null
+++ b/bundles/org.eclipse.emf.ecp.test.common/src/org/eclipse/emf/ecp/test/common/MultiTry.java
@@ -0,0 +1,38 @@
+/*******************************************************************************
+ * Copyright (c) 2011-2019 EclipseSource Muenchen GmbH and others.
+ *
+ * All rights reserved. This program and the accompanying materials
+ * are made available under the terms of the Eclipse Public License v1.0
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/epl-v10.html
+ *
+ * Contributors:
+ * Lucas Koehler - initial API and implementation
+ ******************************************************************************/
+package org.eclipse.emf.ecp.test.common;
+
+import static java.lang.annotation.ElementType.METHOD;
+import static java.lang.annotation.RetentionPolicy.RUNTIME;
+
+import java.lang.annotation.Retention;
+import java.lang.annotation.Target;
+
+/**
+ * Annotation used in conjunction with {@link MultiTryTestRule}. If a {@link MultiTryTestRule} is configured to not
+ * apply to all tests of a test class, only tests annotated with this annotation allow multiple tries. Furthermore, this
+ * annotation allows to increase the number of max tries. The number of max tries is the maximum of the rule's number
+ * and the annotation's number.
+ *
+ * @author Lucas Koehler
+ */
+@Retention(RUNTIME)
+@Target(METHOD)
+public @interface MultiTry {
+	/**
+	 * The maximum tries. The effective number of maximum tries is the <strong>maximum</strong> of this value and the
+	 * number configured in a {@link MultiTryTestRule} instance.
+	 *
+	 * @return The maximum number of retries
+	 */
+	int maxTries() default 1;
+}
diff --git a/bundles/org.eclipse.emf.ecp.test.common/src/org/eclipse/emf/ecp/test/common/MultiTryTestRule.java b/bundles/org.eclipse.emf.ecp.test.common/src/org/eclipse/emf/ecp/test/common/MultiTryTestRule.java
new file mode 100644
index 0000000000..47f1f1e71c
--- /dev/null
+++ b/bundles/org.eclipse.emf.ecp.test.common/src/org/eclipse/emf/ecp/test/common/MultiTryTestRule.java
@@ -0,0 +1,89 @@
+/*******************************************************************************
+ * Copyright (c) 2011-2019 EclipseSource Muenchen GmbH and others.
+ *
+ * All rights reserved. This program and the accompanying materials
+ * are made available under the terms of the Eclipse Public License v1.0
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/epl-v10.html
+ *
+ * Contributors:
+ * Lucas Koehler - initial API and implementation
+ ******************************************************************************/
+package org.eclipse.emf.ecp.test.common;
+
+import java.text.MessageFormat;
+
+import org.junit.rules.TestRule;
+import org.junit.runner.Description;
+import org.junit.runners.model.Statement;
+
+/**
+ * This JUnit rule allows to try unit tests a given number of times. If at least one of the tries succeeds, the test
+ * counts as passed. Every try executes all {@link org.junit.Before Before} and {@link org.junit.After After} methods
+ * of the test. However, it does not re-execute {@link org.junit.BeforeClass BeforeClass} and
+ * {@link org.junit.AfterClass AfterClass} methods.
+ *
+ * @author Lucas Koehler
+ *
+ */
+public class MultiTryTestRule implements TestRule {
+
+	private final int maxTries;
+	private boolean executeAll = true;
+
+	/**
+	 * Create a new {@link MultiTryTestRule} which allows the given number of tries for all test cases of a test class.
+	 * The number of tries is never less than 1 even if the given number of tries is lower.
+	 *
+	 * @param maxTries The maximum number of tries
+	 */
+	public MultiTryTestRule(int maxTries) {
+		this.maxTries = Integer.max(1, maxTries);
+	}
+
+	/**
+	 * Create a new {@link MultiTryTestRule} which allows the given number of tries for test cases of a test class.
+	 * The number of tries is never less than 1 even if the given number of tries is lower.
+	 *
+	 * @param maxTries The maximum number of tries
+	 * @param executeAll <code>true</code> if all test cases of the test class should use multiple tries.
+	 *            <code>false</code> if only test cases annotated with {@link MultiTry} should use multiple tries.
+	 */
+	public MultiTryTestRule(int maxTries, boolean executeAll) {
+		this.maxTries = Integer.max(1, maxTries);
+		this.executeAll = executeAll;
+	}
+
+	@Override
+	public Statement apply(Statement base, Description description) {
+		final MultiTry multiTry = description.getAnnotation(MultiTry.class);
+		if (!executeAll && multiTry == null) {
+			// no annotation and not all tests are multi tried => no multi try for this test
+			return base;
+		}
+
+		final int tries = Integer.max(maxTries, multiTry == null ? 1 : multiTry.maxTries());
+		return new Statement() {
+
+			// CHECKSTYLE.OFF: IllegalCatch
+			@Override
+			public void evaluate() throws Throwable {
+				Throwable lastCaughtThrowable = null;
+				for (int i = 1; i <= tries; i++) {
+					try {
+						base.evaluate();
+						return;
+					} catch (final Throwable t) {
+						System.err
+							.println(MessageFormat.format("{0}: Try {1} failed.", description.getDisplayName(), i)); //$NON-NLS-1$
+						lastCaughtThrowable = t;
+					}
+				}
+				System.err.println(MessageFormat.format("{0} finally failed after {1} tries.", //$NON-NLS-1$
+					description.getDisplayName(), tries));
+				throw lastCaughtThrowable;
+			}
+			// CHECKSTYLE.ON: IllegalCatch
+		};
+	}
+}