HEAD - tests - javac - enabling excuse-based comparison - wip

7 files changed, 462 insertions, 214 deletions

diff --git a/org.eclipse.jdt.core.tests.compiler/src/org/eclipse/jdt/core/tests/compiler/regression/AbstractRegressionTest.java b/org.eclipse.jdt.core.tests.compiler/src/org/eclipse/jdt/core/tests/compiler/regression/AbstractRegressionTest.java
index 70e2d9d128..5fbb540610 100644
--- a/org.eclipse.jdt.core.tests.compiler/src/org/eclipse/jdt/core/tests/compiler/regression/AbstractRegressionTest.java
+++ b/org.eclipse.jdt.core.tests.compiler/src/org/eclipse/jdt/core/tests/compiler/regression/AbstractRegressionTest.java
@@ -300,7 +300,9 @@ protected static class JavacTestOptions {
 			EclipseHasSomeMoreWarnings = RUN_JAVAC ? 
 				new Excuse(MismatchType.EclipseWarningsJavacNone) : null,
 			EclipseWarningConfiguredAsError = RUN_JAVAC ? 
-				new Excuse(MismatchType.EclipseErrorsJavacWarnings | MismatchType.EclipseErrorsJavacNone) : null;
+				new Excuse(MismatchType.EclipseErrorsJavacWarnings | MismatchType.EclipseErrorsJavacNone) : null,
+			JavacCompilesBogusReferencedFileAgain = RUN_JAVAC ? 
+					new Excuse(MismatchType.JavacErrorsEclipseNone) : null;
 	}
 	Excuse excuseFor(JavacCompiler compiler) {
 		return null;
@@ -314,6 +316,8 @@ protected static class JavacTestOptions {
 				new EclipseHasABug(MismatchType.JavacErrorsEclipseWarnings) : null,
 			EclipseBug177715 = RUN_JAVAC ? // https://bugs.eclipse.org/bugs/show_bug.cgi?id=177715
 				new EclipseHasABug(MismatchType.JavacErrorsEclipseNone) : null,
+			EclipseBug207935 = RUN_JAVAC ? // https://bugs.eclipse.org/bugs/show_bug.cgi?id=207935
+				new EclipseHasABug(MismatchType.EclipseErrorsJavacNone) : null,
 			EclipseBug216558 = RUN_JAVAC ? // https://bugs.eclipse.org/bugs/show_bug.cgi?id=216558
 				new EclipseHasABug(MismatchType.JavacErrorsEclipseNone) : null,
 			EclipseBug235550 = RUN_JAVAC ? // https://bugs.eclipse.org/bugs/show_bug.cgi?id=235550
@@ -331,17 +335,37 @@ protected static class JavacTestOptions {
 			super(mismatchType);
 		}
 		public static EclipseJustification
-		// https://bugs.eclipse.org/bugs/show_bug.cgi?id=126712
 			EclipseBug40839 = RUN_JAVAC ? // https://bugs.eclipse.org/bugs/show_bug.cgi?id=40839
 				new EclipseJustification(MismatchType.JavacWarningsEclipseNone) : null,
+			EclipseBug72704 = RUN_JAVAC ? // https://bugs.eclipse.org/bugs/show_bug.cgi?id=72704
+				new EclipseJustification(MismatchType.EclipseErrorsJavacNone) : null,
+			EclipseBug83902 = RUN_JAVAC ? // https://bugs.eclipse.org/bugs/show_bug.cgi?id=83902
+				new EclipseJustification(MismatchType.EclipseWarningsJavacNone) {
+					Excuse excuseFor(JavacCompiler compiler) {
+						return compiler.compliance > ClassFileConstants.JDK1_5 ? this : null;
+					}
+				} : null,
+			EclipseBug83902b = RUN_JAVAC ? // https://bugs.eclipse.org/bugs/show_bug.cgi?id=83902
+				new EclipseJustification(MismatchType.JavacErrorsEclipseWarnings) : null,
 			EclipseBug126712 = RUN_JAVAC ? // https://bugs.eclipse.org/bugs/show_bug.cgi?id=126712 & http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=6342411
 				new EclipseJustification(MismatchType.StandardOutputMismatch) {
 					Excuse excuseFor(JavacCompiler compiler) {
 						return compiler.compliance > ClassFileConstants.JDK1_5 ? this : null;
 					}
-				}: null,
+					// WORK consider adding reversed pivots
+				} : null,
 			EclipseBug126744 = RUN_JAVAC ? // https://bugs.eclipse.org/bugs/show_bug.cgi?id=126744
 				new EclipseJustification(MismatchType.JavacErrorsEclipseNone) : null,
+			EclipseBug180789 = RUN_JAVAC ? // https://bugs.eclipse.org/bugs/show_bug.cgi?id=180789
+				new EclipseJustification(MismatchType.EclipseErrorsJavacWarnings) : null,
+			EclipseBug183211 = RUN_JAVAC ? // https://bugs.eclipse.org/bugs/show_bug.cgi?id=183211
+				new EclipseJustification(MismatchType.JavacErrorsEclipseNone | MismatchType.EclipseErrorsJavacNone) : null,
+			EclipseBug183211b = RUN_JAVAC ? // https://bugs.eclipse.org/bugs/show_bug.cgi?id=183211
+				new EclipseJustification(MismatchType.EclipseErrorsJavacNone) {
+					Excuse excuseFor(JavacCompiler compiler) {
+						return compiler.compliance > ClassFileConstants.JDK1_5 ? this : null;
+					}
+				} : null,
 			EclipseBug185422 = RUN_JAVAC ? // https://bugs.eclipse.org/bugs/show_bug.cgi?id=185422
 				new EclipseJustification(MismatchType.JavacErrorsEclipseNone) : null,
 			EclipseBug234815 = RUN_JAVAC ? // https://bugs.eclipse.org/bugs/show_bug.cgi?id=234815
@@ -426,12 +450,16 @@ protected static class JavacTestOptions {
 			JavacBug6500701 = RUN_JAVAC ? // http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=6500701 & https://bugs.eclipse.org/bugs/show_bug.cgi?id=209779
 				new JavacHasABug(
 					MismatchType.StandardOutputMismatch) : null,
+			JavacBug6557661 = RUN_JAVAC ? // http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=6557661 & https://bugs.eclipse.org/bugs/show_bug.cgi?id=129261
+				new JavacHasABug(
+					MismatchType.EclipseErrorsJavacNone) : null,
 			JavacBug6573446 = RUN_JAVAC ? // http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=6573446 & https://bugs.eclipse.org/bugs/show_bug.cgi?id=190945
 				new JavacHasABug(
 					MismatchType.EclipseErrorsJavacNone) : null,
-			JavacBug6557661 = RUN_JAVAC ? // http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=6557661 & https://bugs.eclipse.org/bugs/show_bug.cgi?id=129261
+			JavacBug6575821 = RUN_JAVAC ? // http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=6575821
 				new JavacHasABug(
-					MismatchType.EclipseErrorsJavacNone) : null;
+					MismatchType.JavacErrorsEclipseNone,
+					ClassFileConstants.JDK1_6, 10 /* 1.6.0_10_b08 or better - maybe before */) : null;
 		// bugs that have been fixed but that we've not identified
 		public static JavacHasABug
 			JavacBugFixed_6_10 = RUN_JAVAC ?
@@ -445,8 +473,11 @@ protected static class JavacTestOptions {
 		// bugs that have neither been fixed nor formally identified but which outcomes are obvious enough to clear any doubts
 		public static JavacHasABug
 			JavacGeneratesByteCodeUponWhichJavaThrowsAnException = RUN_JAVAC ?
-					new JavacHasABug(
-						MismatchType.StandardOutputMismatch) : null;
+				new JavacHasABug(
+					MismatchType.StandardOutputMismatch) : null,
+			JavacThrowsAnException = RUN_JAVAC ?
+				new JavacHasABug(
+					MismatchType.JavacErrorsEclipseNone) : null;
 	}
 }
 
@@ -1323,6 +1354,7 @@ protected static class JavacTestOptions {
 			Util.delete(outputTestDirectory);
 		}
 	}
+	// WORK factorize all runJavac implementations, including overrides
 	protected boolean runJavac(String options, String[] testFileNames, String currentDirectoryPath) {
 		Process compileProcess = null;
 		try {
@@ -1974,6 +2006,10 @@ protected void runNegativeTest(String[] testFiles, String expectedCompilerLog) {
 			exception = e;
 			throw e;
 		} finally {
+			if (expectedCompilerLog != null) {
+				checkCompilerLog(testFiles, requestor, 
+						Util.convertToIndependantLineDelimiter(expectedCompilerLog), exception);
+			}
 			if (exception == null) {
 				if (expectingCompilerErrors) {
 					if (!requestor.hasErrors) {
@@ -1987,10 +2023,6 @@ protected void runNegativeTest(String[] testFiles, String expectedCompilerLog) {
 					}
 				}
 			}
-			if (expectedCompilerLog != null) {
-				checkCompilerLog(testFiles, requestor, 
-						Util.convertToIndependantLineDelimiter(expectedCompilerLog), exception);
-			}
 		}
 		if (!requestor.hasErrors || forceExecution) {
 			String sourceFile = testFiles[0];
@@ -2156,6 +2188,7 @@ protected void runConformTest(
 //
 //		// compiler results
 //		null /* do not check compiler log */,
+//		"----------\n" +  /* expected compiler log */
 //		"" /* expected compiler log */,
 //
 //		// runtime results
diff --git a/org.eclipse.jdt.core.tests.compiler/src/org/eclipse/jdt/core/tests/compiler/regression/AnnotationTest.java b/org.eclipse.jdt.core.tests.compiler/src/org/eclipse/jdt/core/tests/compiler/regression/AnnotationTest.java
index 7e89316b83..4ef0c8603e 100644
--- a/org.eclipse.jdt.core.tests.compiler/src/org/eclipse/jdt/core/tests/compiler/regression/AnnotationTest.java
+++ b/org.eclipse.jdt.core.tests.compiler/src/org/eclipse/jdt/core/tests/compiler/regression/AnnotationTest.java
@@ -3898,7 +3898,8 @@ public class AnnotationTest extends AbstractComparableTest {
 		for (int i = 0, ceil = warnings.length; i < ceil; i++) {
 			customOptions.put(warnings[i], CompilerOptions.WARNING);
 		}
-        this.runNegativeTest(
+        this.runConformTest(
+        	true,
             new String[] {
                 "X.java",
 				"public class X {\n" + 
@@ -3906,6 +3907,8 @@ public class AnnotationTest extends AbstractComparableTest {
 				"  }\n" + 
 				"}\n",
             },
+            null,
+            customOptions,
     		"----------\n" + 
     		"1. WARNING in X.java (at line 1)\n" + 
     		"	public class X {\n" + 
@@ -3923,7 +3926,8 @@ public class AnnotationTest extends AbstractComparableTest {
     		"	                                       ^^^^^\n" + 
     		"Empty block should be documented\n" + 
     		"----------\n",
-			null, true, customOptions);
+			null, null,
+			JavacTestOptions.Excuse.EclipseHasSomeMoreWarnings);
     }       
     // check @SuppressWarning support
     public void test131() {
@@ -4445,6 +4449,7 @@ public void test142c() {
 	raiseDeprecationReduceInvalidJavadocSeverity.put(
 			CompilerOptions.OPTION_ReportInvalidJavadoc, CompilerOptions.WARNING);
     this.runNegativeTest(
+    	true,
         new String[] {
             "X.java",
 			"@SuppressWarnings(\"deprecation\")\n" + 
@@ -4465,6 +4470,8 @@ public void test142c() {
 			"	}	\n" + 
 			"}\n",
         },
+        null,
+        raiseDeprecationReduceInvalidJavadocSeverity,
 		"----------\n" + 
 		"1. ERROR in X.java (at line 2)\n" + 
 		"	public class X extends p.OldStuff {\n" + 
@@ -4476,9 +4483,7 @@ public void test142c() {
 		"	^^^^^^^^^^^\n" + 
 		"The method foo() from the type OldStuff is deprecated\n" + 
 		"----------\n",
-		null,
-		true,
-		raiseDeprecationReduceInvalidJavadocSeverity);
+		JavacTestOptions.Excuse.EclipseWarningConfiguredAsError);
 }        
     public void test143() {
         this.runNegativeTest(
@@ -4712,7 +4717,8 @@ public void test142c() {
     }
     //https://bugs.eclipse.org/bugs/show_bug.cgi?id=98091
     public void test150() {
-        this.runNegativeTest(
+        this.runConformTest(
+        	true,
             new String[] {
                 "X.java",
 				"@SuppressWarnings(\"assertIdentifier\")\n" + 
@@ -4723,7 +4729,9 @@ public void test142c() {
     		"	@SuppressWarnings(\"assertIdentifier\")\n" + 
     		"	                  ^^^^^^^^^^^^^^^^^^\n" + 
     		"Unsupported @SuppressWarnings(\"assertIdentifier\")\n" + 
-    		"----------\n");
+    		"----------\n",
+    		null, null,
+    		JavacTestOptions.Excuse.EclipseHasSomeMoreWarnings);
     }
     //https://bugs.eclipse.org/bugs/show_bug.cgi?id=99009
     public void test151() {
@@ -5108,7 +5116,8 @@ public void test142c() {
 		options.put(CompilerOptions.OPTION_ReportInvalidJavadoc, CompilerOptions.WARNING);
 		options.put(CompilerOptions.OPTION_DocCommentSupport, CompilerOptions.ENABLED);
 		options.put(CompilerOptions.OPTION_ReportInvalidJavadocTagsVisibility, CompilerOptions.PRIVATE);
-	    this.runNegativeTest(
+	    this.runConformTest(
+	    	true,
             new String[] {
                 "X.java",
 				"/**\n" +
@@ -5139,10 +5148,11 @@ public void test142c() {
 				"	public int j;\n" +
 				"}"
             },
+            null,
+            options,
             "",
-			null,
-			true,
-			options
+			null, null,
+			JavacTestOptions.SKIP /* suppressed deprecation related warnings */
 		);
     }
 
@@ -5457,7 +5467,8 @@ public void test142c() {
     public void test169() {
     	Map customOptions = getCompilerOptions();
     	customOptions.put(CompilerOptions.OPTION_ReportNonExternalizedStringLiteral, CompilerOptions.WARNING);
-        this.runNegativeTest(
+        this.runConformTest(
+        	true,
             new String[] {
                 "X.java",
     			"@SuppressWarnings(\"serial\")\n" + 
@@ -5465,13 +5476,16 @@ public void test142c() {
     			"	String s = \"Hello\"; \n" +
     			"}"
             },
+            null,
+            customOptions,
             "----------\n" + 
     		"1. WARNING in X.java (at line 3)\n" + 
     		"	String s = \"Hello\"; \n" + 
     		"	           ^^^^^^^\n" + 
     		"Non-externalized string literal; it should be followed by //$NON-NLS-<n>$\n" + 
     		"----------\n",
-			null, true, customOptions);
+			null, null, 
+			JavacTestOptions.Excuse.EclipseHasSomeMoreWarnings);
     }
     
     // https://bugs.eclipse.org/bugs/show_bug.cgi?id=97220 - variation
@@ -5493,7 +5507,8 @@ public void test142c() {
     public void test171() {
     	Map customOptions = getCompilerOptions();
     	customOptions.put(CompilerOptions.OPTION_ReportNonExternalizedStringLiteral, CompilerOptions.WARNING);
-        this.runNegativeTest(
+        this.runConformTest(
+        	true,
             new String[] {
                 "X.java",
     			"public class X extends Exception {\n" +
@@ -5503,6 +5518,7 @@ public void test142c() {
     			"	String s2 = \"Hello2\"; \n" +
     			"}"
             },
+            null, customOptions,
     		"----------\n" + 
     		"1. WARNING in X.java (at line 1)\n" + 
     		"	public class X extends Exception {\n" + 
@@ -5519,14 +5535,15 @@ public void test142c() {
     		"	            ^^^^^^^^\n" + 
     		"Non-externalized string literal; it should be followed by //$NON-NLS-<n>$\n" + 
     		"----------\n",
-			null, true, customOptions);
+    		null, null, JavacTestOptions.Excuse.EclipseHasSomeMoreWarnings);
     }
     // https://bugs.eclipse.org/bugs/show_bug.cgi?id=97220 - variation
     public void test172() {
     	Map customOptions = getCompilerOptions();
     	customOptions.put(CompilerOptions.OPTION_ReportNonExternalizedStringLiteral, CompilerOptions.WARNING);
-        this.runNegativeTest(
-            new String[] {
+        this.runConformTest(
+        	true,
+        	new String[] {
                 "X.java",
     			"@SuppressWarnings(\"serial\")\n" + 
     			"public class X extends Exception {\n" +
@@ -5536,6 +5553,7 @@ public void test142c() {
     			"	String s2 = \"Hello2\"; \n" +
     			"}"
             },
+            null, customOptions,
     		"----------\n" + 
     		"1. WARNING in X.java (at line 5)\n" + 
     		"	@SuppressWarnings(\"serial\")\n" + 
@@ -5547,13 +5565,14 @@ public void test142c() {
     		"	            ^^^^^^^^\n" + 
     		"Non-externalized string literal; it should be followed by //$NON-NLS-<n>$\n" + 
     		"----------\n",
-			null, true, customOptions);
+			null, null, JavacTestOptions.Excuse.EclipseHasSomeMoreWarnings);
     }
     // https://bugs.eclipse.org/bugs/show_bug.cgi?id=97220 - variation
     public void test173() {
     	Map customOptions = getCompilerOptions();
     	customOptions.put(CompilerOptions.OPTION_ReportNonExternalizedStringLiteral, CompilerOptions.WARNING);
-        this.runNegativeTest(
+        this.runConformTest(
+        	true,
             new String[] {
                 "X.java",
     			"@interface Annot {\n" +
@@ -5567,6 +5586,8 @@ public void test142c() {
     			"	String s2 = \"Hello2\"; \n" +
     			"}"
             },
+            null, 
+            customOptions,
     		"----------\n" + 
     		"1. WARNING in X.java (at line 5)\n" + 
     		"	public class X extends Exception {\n" + 
@@ -5583,7 +5604,7 @@ public void test142c() {
     		"	            ^^^^^^^^\n" + 
     		"Non-externalized string literal; it should be followed by //$NON-NLS-<n>$\n" + 
     		"----------\n",
-			null, true, customOptions);
+			null, null, JavacTestOptions.Excuse.EclipseHasSomeMoreWarnings);
     }
     // https://bugs.eclipse.org/bugs/show_bug.cgi?id=97220 - variation
     public void test174() {
@@ -5602,11 +5623,13 @@ public void test142c() {
     			"   @SuppressWarnings(\"serial\")\n" + 
     			"	String s2 = \"Hello2\"; \n" +
     			"}";
-		this.runNegativeTest(
+		this.runConformTest(
+			true,
             new String[] {
                 "X.java",
     			source
             },
+            null, customOptions,
     		"----------\n" + 
     		"1. WARNING in X.java (at line 11)\n" + 
     		"	@SuppressWarnings(\"serial\")\n" + 
@@ -5618,7 +5641,7 @@ public void test142c() {
     		"	            ^^^^^^^^\n" + 
     		"Non-externalized string literal; it should be followed by //$NON-NLS-<n>$\n" + 
     		"----------\n",
-			null, true, customOptions);
+			null, null, JavacTestOptions.Excuse.EclipseHasSomeMoreWarnings);
     }
     // https://bugs.eclipse.org/bugs/show_bug.cgi?id=97220 - variation
     public void test175() {
diff --git a/org.eclipse.jdt.core.tests.compiler/src/org/eclipse/jdt/core/tests/compiler/regression/AutoBoxingTest.java b/org.eclipse.jdt.core.tests.compiler/src/org/eclipse/jdt/core/tests/compiler/regression/AutoBoxingTest.java
index 4ffc41a8c1..140cb86957 100644
--- a/org.eclipse.jdt.core.tests.compiler/src/org/eclipse/jdt/core/tests/compiler/regression/AutoBoxingTest.java
+++ b/org.eclipse.jdt.core.tests.compiler/src/org/eclipse/jdt/core/tests/compiler/regression/AutoBoxingTest.java
@@ -2979,8 +2979,10 @@ public class AutoBoxingTest extends AbstractComparableTest {
 	}
 	// https://bugs.eclipse.org/bugs/show_bug.cgi?id=84801
 	public void test102() {
-		this.runConformTest(
-			new String[] {
+		runConformTest(
+			// test directory preparation
+			true /* flush output directory */, 
+			new String[] { /* test files */
 				"X.java",
 				"class Cla<A> {\n" + 
 				"	A val;\n" + 
@@ -3019,7 +3021,13 @@ public class AutoBoxingTest extends AbstractComparableTest {
 				"	}\n" + 
 				"}\n"
 			},
-			"001122");
+			// compiler results
+			null /* do not check compiler log */,
+			// runtime results
+			"001122" /* expected output string */,
+			"" /* expected error string */,
+			// javac options
+			JavacTestOptions.JavacHasABug.JavacBugFixed_6_10 /* javac test options */);
 	}
 	// https://bugs.eclipse.org/bugs/show_bug.cgi?id=84801 - variation (check warnings)
 	public void test103() {
@@ -3134,8 +3142,10 @@ public class AutoBoxingTest extends AbstractComparableTest {
 	}
 //https://bugs.eclipse.org/bugs/show_bug.cgi?id=101779
 public void test105() {
-	this.runConformTest(
-		new String[] {
+	runConformTest(
+		// test directory preparation
+		true /* flush output directory */, 
+		new String[] { /* test files */
 			"X.java",
 			"final class Pair<F, S> {\n" + 
 			"	public F first;\n" + 
@@ -3166,12 +3176,20 @@ public void test105() {
 			"	}\n" + 
 			"}\n",
 		},
-		"-2");
+		// compiler results
+		null /* do not check compiler log */,
+		// runtime results
+		"-2" /* expected output string */,
+		"" /* expected error string */,
+		// javac options
+		JavacTestOptions.JavacHasABug.JavacBugFixed_6_10 /* javac test options */);
 }
 //https://bugs.eclipse.org/bugs/show_bug.cgi?id=101779 - variation
 public void test106() {
-	this.runConformTest(
-		new String[] {
+	runConformTest(
+		// test directory preparation
+		true /* flush output directory */, 
+		new String[] { /* test files */
 			"X.java",
 			"class XSuper<T> {\n" + 
 			"	T value;\n" + 
@@ -3192,12 +3210,20 @@ public void test106() {
 			"	}\n" + 
 			"}\n",
 		},
-		"1");
+		// compiler results
+		null /* do not check compiler log */,
+		// runtime results
+		"1" /* expected output string */,
+		"" /* expected error string */,
+		// javac options
+		JavacTestOptions.JavacHasABug.JavacBugFixed_6_10 /* javac test options */);
 }
 //https://bugs.eclipse.org/bugs/show_bug.cgi?id=101779 - variation
 public void test107() {
-	this.runConformTest(
-		new String[] {
+	runConformTest(
+		// test directory preparation
+		true /* flush output directory */, 
+		new String[] { /* test files */
 			"X.java",
 			"class XSuper<T> {\n" + 
 			"	T value;\n" + 
@@ -3218,12 +3244,20 @@ public void test107() {
 			"	}\n" + 
 			"}\n",
 		},
-		"1");
+		// compiler results
+		null /* do not check compiler log */,
+		// runtime results
+		"1" /* expected output string */,
+		"" /* expected error string */,
+		// javac options
+		JavacTestOptions.JavacHasABug.JavacBugFixed_6_10 /* javac test options */);
 }
 //https://bugs.eclipse.org/bugs/show_bug.cgi?id=101779 - variation
 public void test108() {
-	this.runConformTest(
-		new String[] {
+	runConformTest(
+		// test directory preparation
+		true /* flush output directory */, 
+		new String[] { /* test files */
 			"X.java",
 			"class XSuper<T> {\n" + 
 			"	T value;\n" + 
@@ -3244,7 +3278,13 @@ public void test108() {
 			"	}\n" + 
 			"}\n",
 		},
-		"1");
+		// compiler results
+		null /* do not check compiler log */,
+		// runtime results
+		"1" /* expected output string */,
+		"" /* expected error string */,
+		// javac options
+		JavacTestOptions.JavacHasABug.JavacBugFixed_6_10 /* javac test options */);
 }
 //https://bugs.eclipse.org/bugs/show_bug.cgi?id=100043
 public void test109() {
@@ -3280,8 +3320,10 @@ public void test110() {
 }
 //https://bugs.eclipse.org/bugs/show_bug.cgi?id=105524
 public void test111() {
-	this.runConformTest(
-		new String[] {
+	runConformTest(
+		// test directory preparation
+		true /* flush output directory */, 
+		new String[] { /* test files */
 			"X.java",
 			"class Wrapper< T >\n" + 
 			"{\n" + 
@@ -3300,7 +3342,13 @@ public void test111() {
 			"    }\n" + 
 			"}\n",
 		},
-		"");
+		// compiler results
+		null /* do not check compiler log */,
+		// runtime results
+		"" /* expected output string */,
+		"" /* expected error string */,
+		// javac options
+		JavacTestOptions.JavacHasABug.JavacBugFixed_6_10 /* javac test options */);
 }
 //https://bugs.eclipse.org/bugs/show_bug.cgi?id=105284
 public void test112() {
@@ -3498,8 +3546,10 @@ public void test117() {
 
 // Integer array and method with T extends Integer bound
 public void test118() {
-	this.runConformTest(
-		new String[] {
+	runConformTest(
+		// test directory preparation
+		true /* flush output directory */, 
+		new String[] { /* test files */
 			"X.java",
 			"public class X {\n" + 
 			"    public static <T extends Integer> void foo(final T[] p) {\n" +
@@ -3512,13 +3562,21 @@ public void test118() {
 			"    }\n" + 
 			"}",
 		},
-		"1");
+		// compiler results
+		null /* do not check compiler log */,
+		// runtime results
+		"1" /* expected output string */,
+		"" /* expected error string */,
+		// javac options
+		JavacTestOptions.JavacHasABug.JavacBug6575821 /* javac test options */);
 }
 
 // Integer as member of a parametrized class
 public void test119() {
-	this.runConformTest(
-		new String[] {
+	runConformTest(
+		// test directory preparation
+		true /* flush output directory */, 
+		new String[] { /* test files */
 			"X.java",
 			"public class X<T> {\n" + 
 			"    T m;\n" + 
@@ -3532,7 +3590,13 @@ public void test119() {
 			"    }\n" + 
 			"}",
 		},
-		"1");
+		// compiler results
+		null /* do not check compiler log */,
+		// runtime results
+		"1" /* expected output string */,
+		"" /* expected error string */,
+		// javac options
+		JavacTestOptions.JavacHasABug.JavacBugFixed_6_10 /* javac test options */);
 }
 //https://bugs.eclipse.org/bugs/show_bug.cgi?id=137918
 public void test120() {
diff --git a/org.eclipse.jdt.core.tests.compiler/src/org/eclipse/jdt/core/tests/compiler/regression/Compliance_1_5.java b/org.eclipse.jdt.core.tests.compiler/src/org/eclipse/jdt/core/tests/compiler/regression/Compliance_1_5.java
index 71ea29e691..e3b87e45de 100644
--- a/org.eclipse.jdt.core.tests.compiler/src/org/eclipse/jdt/core/tests/compiler/regression/Compliance_1_5.java
+++ b/org.eclipse.jdt.core.tests.compiler/src/org/eclipse/jdt/core/tests/compiler/regression/Compliance_1_5.java
@@ -1168,23 +1168,31 @@ public void test036() {
 public void test037() {
 	Map customOptions = getCompilerOptions();
 	customOptions.put(CompilerOptions.OPTION_TaskTags, "TODO:");
-	this.runNegativeTest(
-		new String[] {
+	runConformTest(
+		// test directory preparation
+		true /* flush output directory */, 
+		new String[] { /* test files */
 			"p/X.java",
 			"package p;	\n"+
 			"public class X {\n"+
 			"}\n"+
 			"// TODO: something"
 		},
-		"----------\n" + 
+		// compiler options
+		null /* no class libraries */,
+		customOptions /* custom options */,
+		// compiler results
+		"----------\n" +  /* expected compiler log */
 		"1. WARNING in p\\X.java (at line 4)\n" + 
 		"	// TODO: something\n" + 
 		"	   ^^^^^^^^^^^^^^^\n" + 
 		"TODO: something\n" + 
 		"----------\n",
-		null,
-		true,
-		customOptions);
+		// runtime results
+		null /* do not check output string */,
+		null /* do not check error string */,
+		// javac options
+		JavacTestOptions.Excuse.EclipseHasSomeMoreWarnings /* javac test options */);	
 }
 
 /*
@@ -1196,20 +1204,28 @@ public void test037() {
 public void test038() {
 	Map customOptions = getCompilerOptions();
 	customOptions.put(CompilerOptions.OPTION_TaskTags, "TODO:");
-	this.runNegativeTest(
-		new String[] {
+	runConformTest(
+		// test directory preparation
+		true /* flush output directory */, 
+		new String[] { /* test files */
 			"X.java",
 			"// TODO: something"
 		},
-		"----------\n" + 
+		// compiler options
+		null /* no class libraries */,
+		customOptions /* custom options */,
+		// compiler results
+		"----------\n" +  /* expected compiler log */
 		"1. WARNING in X.java (at line 1)\n" + 
 		"	// TODO: something\n" + 
 		"	   ^^^^^^^^^^^^^^^\n" + 
 		"TODO: something\n" + 
 		"----------\n",
-		null,
-		true,
-		customOptions);
+		// runtime results
+		null /* do not check output string */,
+		null /* do not check error string */,
+		// javac options
+		JavacTestOptions.Excuse.EclipseHasSomeMoreWarnings /* javac test options */);	
 }
 
 /*
@@ -1777,8 +1793,10 @@ public void test054() {
 	);
 }
 public void test055() {
-	this.runNegativeTest(
-		new String[] {
+	runConformTest(
+		// test directory preparation
+		true /* flush output directory */, 
+		new String[] { /* test files */
 			"p/X.java",
 			"package p;\n" + 
 			"public class X {\n" + 
@@ -1806,8 +1824,9 @@ public void test055() {
 			"  new X().new Y().foo(8);\n" + 
 			"}\n" + 
 			"}",
-		}, 
-		"----------\n" + 
+		},
+		// compiler results
+		"----------\n" +  /* expected compiler log */
 		"1. WARNING in p\\X.java (at line 11)\n" + 
 		"	void foo(int a);\n" + 
 		"	             ^\n" + 
@@ -1817,8 +1836,12 @@ public void test055() {
 		"	public void foo(int a) {\n" + 
 		"	                    ^\n" + 
 		"The parameter a is hiding a field from type X.I1\n" + 
-		"----------\n"
-	);
+		"----------\n",
+		// runtime results
+		null /* do not check output string */,
+		null /* do not check error string */,
+		// javac options
+		JavacTestOptions.Excuse.EclipseHasSomeMoreWarnings /* javac test options */);	
 }
 
 public void test056() {
@@ -1899,8 +1922,10 @@ public void test058() {
 }
 
 public void test059() {
-	this.runNegativeTest(
-		new String[] {
+	runConformTest(
+		// test directory preparation
+		true /* flush output directory */, 
+		new String[] { /* test files */
 			"p/FieldQualification.java",
 			"package p;\n" + 
 			"public class FieldQualification {\n" + 
@@ -1919,8 +1944,9 @@ public void test059() {
 			"  };\n" + 
 			"}\n" + 
 			"}",
-		}, 
-		"----------\n" + 
+		},
+		// compiler results
+		"----------\n" +  /* expected compiler log */
 		"1. WARNING in p\\FieldQualification.java (at line 5)\n" +
 		"	class Local {\n" +
 		"	      ^^^^^\n" +
@@ -1940,7 +1966,12 @@ public void test059() {
 		"	void foo() {\n" + 
 		"	     ^^^^^\n" + 
 		"The method foo() from the type Local is never used locally\n" + 
-		"----------\n");
+		"----------\n",
+		// runtime results
+		null /* do not check output string */,
+		null /* do not check error string */,
+		// javac options
+		JavacTestOptions.Excuse.EclipseHasSomeMoreWarnings /* javac test options */);	
 }
 
 public void test060() {
@@ -1977,8 +2008,10 @@ public void test060() {
  * http://bugs.eclipse.org/bugs/show_bug.cgi?id=32342
  */
 public void test061() {
-	this.runNegativeTest(
-		new String[] {
+	runConformTest(
+		// test directory preparation
+		true /* flush output directory */, 
+		new String[] { /* test files */
 			"p/X.java", //======================
 			"package p;	\n" +
 			"public class X extends q.Y {	\n" +
@@ -1994,12 +2027,18 @@ public void test061() {
 			"	public static class Z {}	\n" +
 			"}	\n"
 		},
-		"----------\n" + 
+		// compiler results
+		"----------\n" +  /* expected compiler log */
 		"1. WARNING in q\\Y.java (at line 3)\n" + 
 		"	private static class X {}	\n" + 
 		"	                     ^\n" + 
 		"The type Y.X is never used locally\n" + 
-		"----------\n");
+		"----------\n",
+		// runtime results
+		null /* do not check output string */,
+		null /* do not check error string */,
+		// javac options
+		JavacTestOptions.Excuse.EclipseHasSomeMoreWarnings /* javac test options */);	
 }
 /*
  * http://bugs.eclipse.org/bugs/show_bug.cgi?id=11435
@@ -2515,7 +2554,8 @@ public void test076() {
 		"	* @throws Problem \n" + 
 		"	          ^^^^^^^\n" + 
 		"Javadoc: Exception Problem is not declared\n" + 
-		"----------\n"	);
+		"----------\n",
+		JavacTestOptions.Excuse.EclipseWarningConfiguredAsError);
 }
 /**
  * Test fix bug 58069 for method.
@@ -2758,8 +2798,10 @@ public void test084() {
  * Test unused import with static
  */
 public void test085() {
-	this.runNegativeTest(
-		new String[] {
+	runConformTest(
+		// test directory preparation
+		true /* flush output directory */, 
+		new String[] { /* test files */
 			"A.java",
 			"import static j.l.S.*;\n" + 
 				"import static j.l.S.in;\n" + 
@@ -2773,17 +2815,23 @@ public void test085() {
 				"	public static int in;\n" + 
 				"}\n"
 		},
-		"----------\n" + 
-			"1. WARNING in A.java (at line 1)\n" + 
-			"	import static j.l.S.*;\n" + 
-			"	              ^^^^^\n" + 
-			"The import j.l.S is never used\n" + 
-			"----------\n" + 
-			"2. WARNING in A.java (at line 2)\n" + 
-			"	import static j.l.S.in;\n" + 
-			"	              ^^^^^^^^\n" + 
-			"The import j.l.S.in is never used\n" + 
-			"----------\n");
+		// compiler results
+		"----------\n" +  /* expected compiler log */
+		"1. WARNING in A.java (at line 1)\n" + 
+		"	import static j.l.S.*;\n" + 
+		"	              ^^^^^\n" + 
+		"The import j.l.S is never used\n" + 
+		"----------\n" + 
+		"2. WARNING in A.java (at line 2)\n" + 
+		"	import static j.l.S.in;\n" + 
+		"	              ^^^^^^^^\n" + 
+		"The import j.l.S.in is never used\n" + 
+		"----------\n",
+		// runtime results
+		null /* do not check output string */,
+		null /* do not check error string */,
+		// javac options
+		JavacTestOptions.Excuse.EclipseHasSomeMoreWarnings /* javac test options */);	
 }
 /**
  * Test invalid static import syntax
@@ -3437,7 +3485,7 @@ public void test105() {
 			"    void foo(Y p) { }\n" + 
 			"}\n"
 		},
-		"");
+		JavacTestOptions.JavacHasABug.JavacBugFixed_6_10);
 }
 //https://bugs.eclipse.org/bugs/show_bug.cgi?id=79798
 public void test106() {
diff --git a/org.eclipse.jdt.core.tests.compiler/src/org/eclipse/jdt/core/tests/compiler/regression/EnumTest.java b/org.eclipse.jdt.core.tests.compiler/src/org/eclipse/jdt/core/tests/compiler/regression/EnumTest.java
index 4cb0d12d6a..85db250c77 100644
--- a/org.eclipse.jdt.core.tests.compiler/src/org/eclipse/jdt/core/tests/compiler/regression/EnumTest.java
+++ b/org.eclipse.jdt.core.tests.compiler/src/org/eclipse/jdt/core/tests/compiler/regression/EnumTest.java
@@ -1055,7 +1055,8 @@ public void test034() {
 			"	* @param x Invalid tag\n" +
 			"	   ^^^^^\n" +
 			"Javadoc: Unexpected tag\n" +
-			"----------\n"
+			"----------\n",
+			JavacTestOptions.Excuse.EclipseWarningConfiguredAsError
 	);
 }
 public void test035() {
@@ -1106,7 +1107,8 @@ public void test036() {
 			"	* @see #VALIDE\n" + 
 			"	        ^^^^^^\n" + 
 			"Javadoc: VALIDE cannot be resolved or is not a field\n" + 
-			"----------\n"
+			"----------\n",
+			JavacTestOptions.Excuse.EclipseWarningConfiguredAsError
 	);
 }
 public void test037() {
@@ -1150,7 +1152,8 @@ public void test038() {
 			"	public void foo() {}\n" + 
 			"	            ^^^^^\n" + 
 			"Javadoc: Missing comment for public declaration\n" + 
-			"----------\n"
+			"----------\n",
+			JavacTestOptions.Excuse.EclipseWarningConfiguredAsError
 	);
 }
 public void test039() {
@@ -1188,7 +1191,8 @@ public void test039() {
 			"	* @param x Invalid tag\n" +
 			"	   ^^^^^\n" +
 			"Javadoc: Unexpected tag\n" +
-			"----------\n"
+			"----------\n",
+			JavacTestOptions.Excuse.EclipseWarningConfiguredAsError
 	);
 }
 public void test040() {
@@ -1279,7 +1283,8 @@ public void test041() {
 		"	public String val(Object x) { return x.toString(); }\n" + 
 		"	                         ^\n" + 
 		"Javadoc: Missing tag for parameter x\n" + 
-		"----------\n"
+		"----------\n",
+		JavacTestOptions.Excuse.EclipseWarningConfiguredAsError
 	);
 }
 public void test042() {
@@ -3159,7 +3164,8 @@ public void test100() {
 		"	private final X thisOne = anEnumValue;\n" + 
 		"	                          ^^^^^^^^^^^\n" + 
 		"Cannot refer to the static enum field X.anEnumValue within an initializer\n" + 
-		"----------\n");
+		"----------\n",
+		JavacTestOptions.JavacHasABug.JavacBugFixed_6_10);
 }	
 // https://bugs.eclipse.org/bugs/show_bug.cgi?id=91761
 public void test101() {
@@ -4077,7 +4083,8 @@ public void test120() {
 		"	final X a2 = B.A;\n" + 
 		"	               ^\n" + 
 		"The static field X.A should be accessed in a static way\n" + 
-		"----------\n");
+		"----------\n",
+		JavacTestOptions.JavacHasABug.JavacBugFixed_6_10);
 }			
 //https://bugs.eclipse.org/bugs/show_bug.cgi?id=92165
 public void test121() {
@@ -4616,7 +4623,8 @@ public void test134() {
 		"	System.out.printf(\"After the %s constructor\\n\",INITIAL);\n" + 
 		"	                                               ^^^^^^^\n" + 
 		"Cannot refer to the static enum field X.INITIAL within an initializer\n" + 
-		"----------\n");
+		"----------\n",
+		JavacTestOptions.JavacHasABug.JavacBugFixed_6_10);
 }
 // https://bugs.eclipse.org/bugs/show_bug.cgi?id=149562
 // a default case is required to consider that b is initialized (in case E 
@@ -4983,23 +4991,29 @@ public void test144() {
 public void test145() {
 	Map options = getCompilerOptions();
 	options.put(CompilerOptions.OPTION_ReportNonStaticAccessToStatic, CompilerOptions.ERROR);
-	this.runNegativeTest(
-		new String[] {
-				"EnumA.java",
-				"public enum EnumA {\r\n" + 
-				"  B1,\r\n" + 
-				"  B2;\r\n" + 
-				"  public void foo(){}\r\n" + 
-				"}",
-				"ClassC.java",
-				"public class ClassC {\r\n" + 
-				"  void bar() {\r\n" + 
-				"    EnumA.B1.B1.foo();\r\n" + 
-				"    EnumA.B1.B2.foo();\r\n" + 
-				"  }\r\n" + 
-				"}"
+	runNegativeTest(
+		// test directory preparation
+		true /* flush output directory */, 
+		new String[] { /* test files */
+			"EnumA.java",
+			"public enum EnumA {\r\n" + 
+			"  B1,\r\n" + 
+			"  B2;\r\n" + 
+			"  public void foo(){}\r\n" + 
+			"}",
+			"ClassC.java",
+			"public class ClassC {\r\n" + 
+			"  void bar() {\r\n" + 
+			"    EnumA.B1.B1.foo();\r\n" + 
+			"    EnumA.B1.B2.foo();\r\n" + 
+			"  }\r\n" + 
+			"}"
 		},
-		"----------\n" + 
+		// compiler options
+		null /* no class libraries */,
+		options /* custom options */,
+		// compiler results
+		"----------\n" + /* expected compiler log */
 		"1. ERROR in ClassC.java (at line 3)\n" + 
 		"	EnumA.B1.B1.foo();\n" + 
 		"	         ^^\n" + 
@@ -5010,9 +5024,8 @@ public void test145() {
 		"	         ^^\n" + 
 		"The static field EnumA.B2 should be accessed in a static way\n" + 
 		"----------\n",
-		null,
-		true,
-		options);
+		// javac options
+		JavacTestOptions.Excuse.EclipseWarningConfiguredAsError /* javac test options */);	
 }
 //https://bugs.eclipse.org/bugs/show_bug.cgi?id=207915
 public void test146() {
@@ -5068,20 +5081,25 @@ public void test147() {
 			true, // generate output
 			false,
 			false);			
-	this.runConformTest(
-		new String[] {
-				"Y.java",
-				"import p.X;\n" +
-				"public class Y {\n" + 
-				"	public static void main(String[] args) {\n" + 
-				"		System.out.println(X.E.SUCCESS);\n" + 
-				"	}\n" + 
-				"}\n"
+	runConformTest(
+		// test directory preparation
+		false /* do not flush output directory */,
+		new String[] { /* test files */
+			"Y.java",
+			"import p.X;\n" +
+			"public class Y {\n" + 
+			"	public static void main(String[] args) {\n" + 
+			"		System.out.println(X.E.SUCCESS);\n" + 
+			"	}\n" + 
+			"}\n"
 		},
-		"null",
-		null,
-		false,
-		null);
+		// compiler results
+		null /* do not check compiler log */,
+		// runtime results
+		"null" /* expected output string */,
+		"" /* expected error string */,
+		// javac options
+		JavacTestOptions.Excuse.JavacCompilesBogusReferencedFileAgain /* javac test options */);	
 }
 //https://bugs.eclipse.org/bugs/show_bug.cgi?id=227502 - variation
 public void test148() {
@@ -5112,20 +5130,25 @@ public void test148() {
 			true, // generate output
 			false,
 			false);			
-	this.runConformTest(
-		new String[] {
-				"Y.java",
-				"import p.X;\n" +
-				"public class Y {\n" + 
-				"	public static void main(String[] args) {\n" + 
-				"		System.out.println(X.E.SUCCESS);\n" + 
-				"	}\n" + 
-				"}\n"
+	runConformTest(
+		// test directory preparation
+		false /* do not flush output directory */,
+		new String[] { /* test files */
+			"Y.java",
+			"import p.X;\n" +
+			"public class Y {\n" + 
+			"	public static void main(String[] args) {\n" + 
+			"		System.out.println(X.E.SUCCESS);\n" + 
+			"	}\n" + 
+			"}\n"
 		},
-		"null",
-		null,
-		false,
-		null);
+		// compiler results
+		null /* do not check compiler log */,
+		// runtime results
+		"null" /* expected output string */,
+		"" /* expected error string */,
+		// javac options
+		JavacTestOptions.Excuse.JavacCompilesBogusReferencedFileAgain /* javac test options */);	
 }
 //https://bugs.eclipse.org/bugs/show_bug.cgi?id=227502 - variation
 public void test149() throws Exception {
@@ -5286,7 +5309,8 @@ public void test153() {
 		"	TestEnum.test=10;\n" + 
 		"	         ^^^^\n" + 
 		"Cannot refer to the static enum field TestEnum.test within an initializer\n" + 
-		"----------\n");
+		"----------\n",
+		JavacTestOptions.JavacHasABug.JavacBugFixed_6_10);
 }
 //https://bugs.eclipse.org/bugs/show_bug.cgi?id=228109 - variation
 public void test154() {
@@ -5312,7 +5336,8 @@ public void test154() {
 		"	TestEnum2.test=11;\n" + 
 		"	          ^^^^\n" + 
 		"Cannot refer to the static enum field TestEnum2.test within an initializer\n" + 
-		"----------\n");
+		"----------\n",
+		JavacTestOptions.JavacHasABug.JavacBugFixed_6_10);
 }
 //https://bugs.eclipse.org/bugs/show_bug.cgi?id=228109 - variation
 public void test155() {
@@ -5390,7 +5415,8 @@ public void test157() {
 		"	int field = Foo.val;\n" + 
 		"	                ^^^\n" + 
 		"Cannot refer to the static enum field Foo.val within an initializer\n" + 
-		"----------\n");
+		"----------\n",
+		JavacTestOptions.JavacHasABug.JavacBugFixed_6_10);
 }
 //https://bugs.eclipse.org/bugs/show_bug.cgi?id=228109 - variation
 public void test158() {
diff --git a/org.eclipse.jdt.core.tests.compiler/src/org/eclipse/jdt/core/tests/compiler/regression/MethodVerifyTest.java b/org.eclipse.jdt.core.tests.compiler/src/org/eclipse/jdt/core/tests/compiler/regression/MethodVerifyTest.java
index 91060f2489..6278b219a9 100644
--- a/org.eclipse.jdt.core.tests.compiler/src/org/eclipse/jdt/core/tests/compiler/regression/MethodVerifyTest.java
+++ b/org.eclipse.jdt.core.tests.compiler/src/org/eclipse/jdt/core/tests/compiler/regression/MethodVerifyTest.java
@@ -1688,7 +1688,8 @@ public class MethodVerifyTest extends AbstractComparableTest {
 		);
 	}
 	public void test025e() { // 81618
-		this.runNegativeTest(
+		this.runConformTest(
+			true,
 			new String[] {
 				"X.java",
 				"interface X<T extends X> { T x(); }\n" +
@@ -1715,11 +1716,14 @@ public class MethodVerifyTest extends AbstractComparableTest {
 			"	abstract class Z implements X { public abstract X x(); }\n" + 
 			"	                                                ^\n" + 
 			"X is a raw type. References to generic type X<T> should be parameterized\n" + 
-			"----------\n"
+			"----------\n",
+			null, null,
+			JavacTestOptions.Excuse.EclipseHasSomeMoreWarnings
 		);
 	}		
 	public void test025f() { // 81618
-		this.runNegativeTest(
+		this.runConformTest(
+			true,
 			new String[] {
 				"X.java",
 				"interface X<T extends X> { T[] x(); }\n" +
@@ -1746,7 +1750,9 @@ public class MethodVerifyTest extends AbstractComparableTest {
 			"	abstract class Z implements X { public abstract X[] x(); }\n" + 
 			"	                                                ^\n" + 
 			"X is a raw type. References to generic type X<T> should be parameterized\n" + 
-			"----------\n"
+			"----------\n",
+			null, null,
+			JavacTestOptions.Excuse.EclipseHasSomeMoreWarnings
 		);
 	}
 
@@ -2731,7 +2737,8 @@ public class MethodVerifyTest extends AbstractComparableTest {
 
 	// https://bugs.eclipse.org/bugs/show_bug.cgi?id=83902
 	public void test041() { // inherited cases for bridge methods, varargs clashes, return type conversion checks
-		this.runNegativeTest(
+		runConformTest(
+			true,
 			new String[] {
 				"X.java",
 				"public class X { public void foo(String... n) {} }\n" +
@@ -2743,13 +2750,17 @@ public class MethodVerifyTest extends AbstractComparableTest {
 			"	class Y extends X implements I { }\n" + 
 			"	      ^\n" + 
 			"Varargs methods should only override or be overridden by other varargs methods unlike X.foo(String...) and I.foo(String[])\n" + 
-			"----------\n"
+			"----------\n",
+			null,
+			null,
+			JavacTestOptions.EclipseJustification.EclipseBug83902
 			// warning: foo(java.lang.String...) in X cannot implement foo(java.lang.String[]) in I; overridden method has no '...'
 		);
 	}
 	// https://bugs.eclipse.org/bugs/show_bug.cgi?id=83902
 	public void test041a() { // inherited cases for bridge methods, varargs clashes, return type conversion checks
-		this.runNegativeTest(
+		this.runConformTest(
+			true,
 			new String[] {
 				"X.java",
 				"public class X { public void foo(String[] n) {} }\n" +
@@ -2761,7 +2772,10 @@ public class MethodVerifyTest extends AbstractComparableTest {
 			"	class Y extends X implements I { }\n" + 
 			"	      ^\n" + 
 			"Varargs methods should only override or be overridden by other varargs methods unlike X.foo(String[]) and I.foo(String...)\n" + 
-			"----------\n"
+			"----------\n",
+			null,
+			null,
+			JavacTestOptions.EclipseJustification.EclipseBug83902
 			// warning: foo(java.lang.String[]) in X cannot implement foo(java.lang.String...) in I; overriding method is missing '...'
 		);
 	}
@@ -2811,7 +2825,8 @@ public class MethodVerifyTest extends AbstractComparableTest {
 	}
 	// https://bugs.eclipse.org/bugs/show_bug.cgi?id=83902
 	public void test041d() { // inherited cases for bridge methods, varargs clashes, return type conversion checks
-		this.runNegativeTest(
+		this.runConformTest(
+			true,
 			new String[] {
 				"X.java",
 				"public class X { public Object foo() { return null; } }\n" +
@@ -2823,7 +2838,9 @@ public class MethodVerifyTest extends AbstractComparableTest {
 			"	class Y extends X implements I { }\n" + 
 			"	      ^\n" + 
 			"Type safety: The return type Object for foo() from the type X needs unchecked conversion to conform to T from the type I\n" + 
-			"----------\n"
+			"----------\n",
+			null, null,
+			JavacTestOptions.EclipseJustification.EclipseBug83902b
 			// NOTE: javac issues an error & a warning which contradict each other
 			// if the method Object foo() is implemented in Y then only the warning is issued, so X should be allowed to implement the method
 			// Y is not abstract and does not override abstract method <T>foo() in I
@@ -2866,7 +2883,8 @@ public class MethodVerifyTest extends AbstractComparableTest {
 			"	abstract class E<A, B> extends C<A> implements I<B> {}\n" + 
 			"	               ^\n" + 
 			"Name clash: The method id(A) of type C<A> has the same erasure as id(B) of type I<B> but does not override it\n" + 
-			"----------\n"
+			"----------\n",
+			JavacTestOptions.EclipseJustification.EclipseBug72704
 			// javac won't report it until C.id() is made concrete or implemented in E
 		);
 	}
@@ -4674,7 +4692,8 @@ public class MethodVerifyTest extends AbstractComparableTest {
 	}
 	//https://bugs.eclipse.org/bugs/show_bug.cgi?id=101049
 	public void test070() {
-		this.runNegativeTest(
+		this.runConformTest(
+			true,
 			new String[] {
 				"BooleanFactory.java",
 				"interface Factory<T> {\n" +
@@ -4691,7 +4710,9 @@ public class MethodVerifyTest extends AbstractComparableTest {
 			"	public <U extends Boolean> U create(Class<U> c) {\n" + 
 			"	                  ^^^^^^^\n" + 
 			"The type parameter U should not be bounded by the final type Boolean. Final types cannot be further extended\n" + 
-			"----------\n"
+			"----------\n",
+			null, null,
+			JavacTestOptions.Excuse.EclipseHasSomeMoreWarnings
 		);
 	}
 	//https://bugs.eclipse.org/bugs/show_bug.cgi?id=107045
@@ -5120,6 +5141,7 @@ public class MethodVerifyTest extends AbstractComparableTest {
 		customOptions.put(CompilerOptions.OPTION_Source, CompilerOptions.VERSION_1_4);
 		customOptions.put(CompilerOptions.OPTION_TargetPlatform, CompilerOptions.VERSION_1_4);
 		this.runNegativeTest(
+			true,
 			new String[] {
 				"X.java",
 				"public abstract class X implements IAppendable {\n" + 
@@ -5132,15 +5154,15 @@ public class MethodVerifyTest extends AbstractComparableTest {
 				"	IAppendable append(char c);\n" + 
 				"}\n",
 			},
+			null,
+			customOptions,
 			"----------\n" + 
 			"1. ERROR in X.java (at line 2)\n" + 
 			"	public X append(char c) {\n" + 
 			"	       ^\n" + 
 			"The return type is incompatible with IAppendable.append(char)\n" + 
 			"----------\n",
-		null,
-		true,
-		customOptions);		
+			JavacTestOptions.SKIP /* we are altering the compatibility settings */);		
 	}			
 	//https://bugs.eclipse.org/bugs/show_bug.cgi?id=81222
 	public void test079() {
@@ -7014,8 +7036,10 @@ public void test113() {
 public void test114() {
 	Map options = this.getCompilerOptions();
 	options.put(CompilerOptions.OPTION_ReportOverridingMethodWithoutSuperInvocation, CompilerOptions.ERROR);
-	this.runNegativeTest(
-		new String[] {
+	runNegativeTest(
+		// test directory preparation
+		true /* flush output directory */, 
+		new String[] { /* test files */
 			"X.java",
 			"class Y {\n" + 
 			"  void foo() {}\n" + 
@@ -7026,16 +7050,18 @@ public void test114() {
 			"  }\n" + 
 			"}"
 		},
-		"----------\n" + 
+		// compiler options
+		null /* no class libraries */,
+		options /* custom options */,
+		// compiler results
+		"----------\n" + /* expected compiler log */
 		"1. ERROR in X.java (at line 6)\n" + 
 		"	void foo() {\n" + 
 		"	     ^^^^^\n" + 
 		"The method X.foo() is overriding a method without making a super invocation\n" + 
 		"----------\n",
-		null,
-		true,
-		options
-	);
+		// javac options
+		JavacTestOptions.Excuse.EclipseWarningConfiguredAsError /* javac test options */);	
 }
 //https://bugs.eclipse.org/bugs/show_bug.cgi?id=156736
 public void test115() {
@@ -7122,8 +7148,10 @@ public void test116() {
 public void test117() {
 	Map options = this.getCompilerOptions();
 	options.put(CompilerOptions.OPTION_ReportOverridingMethodWithoutSuperInvocation, CompilerOptions.ERROR);
-	this.runNegativeTest(
-		new String[] {
+	runNegativeTest(
+		// test directory preparation
+		true /* flush output directory */, 
+		new String[] { /* test files */
 			"X.java",
 			"class Y {\n" + 
 			"  Object foo() {\n" +
@@ -7142,7 +7170,11 @@ public void test117() {
 			"  }\n" + 
 			"}"
 		},
-		"----------\n" + 
+		// compiler options
+		null /* no class libraries */,
+		options /* custom options */,
+		// compiler results
+		"----------\n" + /* expected compiler log */
 		"1. ERROR in X.java (at line 8)\n" + 
 		"	Object foo() {\n" + 
 		"	       ^^^^^\n" + 
@@ -7153,17 +7185,17 @@ public void test117() {
 		"	       ^^^^^\n" + 
 		"The method new Y(){}.foo() is overriding a method without making a super invocation\n" + 
 		"----------\n",
-		null,
-		true,
-		options
-	);
+		// javac options
+		JavacTestOptions.Excuse.EclipseWarningConfiguredAsError /* javac test options */);	
 }
 //https://bugs.eclipse.org/bugs/show_bug.cgi?id=156736
 public void test118() {
 	Map options = this.getCompilerOptions();
 	options.put(CompilerOptions.OPTION_ReportOverridingMethodWithoutSuperInvocation, CompilerOptions.ERROR);
-	this.runNegativeTest(
-		new String[] {
+	runNegativeTest(
+		// test directory preparation
+		true /* flush output directory */, 
+		new String[] { /* test files */
 			"X.java",
 			"class Y<E> {\n" + 
 			"	<U extends E> U foo() {\n" + 
@@ -7178,23 +7210,27 @@ public void test118() {
 			"	}\n" + 
 			"}"
 		},
-		"----------\n" + 
+		// compiler options
+		null /* no class libraries */,
+		options /* custom options */,
+		// compiler results
+		"----------\n" + /* expected compiler log */
 		"1. ERROR in X.java (at line 9)\n" + 
 		"	<V extends T> V foo() {\n" + 
 		"	                ^^^^^\n" + 
 		"The method X<T>.foo() is overriding a method without making a super invocation\n" + 
 		"----------\n",
-		null,
-		true,
-		options
-	);
+		// javac options
+		JavacTestOptions.Excuse.EclipseWarningConfiguredAsError /* javac test options */);	
 }
 //https://bugs.eclipse.org/bugs/show_bug.cgi?id=156736
 public void test119() {
 	Map options = this.getCompilerOptions();
 	options.put(CompilerOptions.OPTION_ReportOverridingMethodWithoutSuperInvocation, CompilerOptions.ERROR);
-	this.runNegativeTest(
-		new String[] {
+	runNegativeTest(
+		// test directory preparation
+		true /* flush output directory */, 
+		new String[] { /* test files */
 			"X.java",
 			"class Y<E> {\n" + 
 			"	E foo() {\n" + 
@@ -7209,16 +7245,18 @@ public void test119() {
 			"	}\n" + 
 			"}"
 		},
-		"----------\n" + 
+		// compiler options
+		null /* no class libraries */,
+		options /* custom options */,
+		// compiler results
+		"----------\n" + /* expected compiler log */
 		"1. ERROR in X.java (at line 9)\n" + 
 		"	T foo() {\n" + 
 		"	  ^^^^^\n" + 
 		"The method X<T>.foo() is overriding a method without making a super invocation\n" + 
 		"----------\n",
-		null,
-		true,
-		options
-	);
+		// javac options
+		JavacTestOptions.Excuse.EclipseWarningConfiguredAsError /* javac test options */);	
 }
 //https://bugs.eclipse.org/bugs/show_bug.cgi?id=161541
 public void test120() {
@@ -7357,7 +7395,7 @@ public void _test124() {
 		"ab");
 }
 // https://bugs.eclipse.org/bugs/show_bug.cgi?id=150655
-// ** variant
+// variant
 public void test125() {
 	this.runNegativeTest(
 		new String[] {
@@ -7378,10 +7416,11 @@ public void test125() {
 		"	return one + X.<String>choose(one, two);\n" + 
 		"	                       ^^^^^^\n" + 
 		"The method choose(String, String) is ambiguous for the type X\n" +  
-		"----------\n");
+		"----------\n",
+		JavacTestOptions.EclipseHasABug.EclipseBug207935);
 }
 // https://bugs.eclipse.org/bugs/show_bug.cgi?id=150655
-// ** variant
+// variant
 // https://bugs.eclipse.org/bugs/show_bug.cgi?id=176171
 // deprecated by GenericTypeTest#test1203.
 //public void _test126() {
@@ -7494,7 +7533,7 @@ public void test129() {
 	);
 }
 // https://bugs.eclipse.org/bugs/show_bug.cgi?id=180789
-// ** variant - Z<Object> is not a subtype of Z<U>, and |Z<U>| = Z, not Z<Object>
+// variant - Z<Object> is not a subtype of Z<U>, and |Z<U>| = Z, not Z<Object>
 public void test130() {
 	this.runNegativeTest(
 		new String[] {
@@ -7512,7 +7551,8 @@ public void test130() {
 		"	public Z<Object> foo(Object o, Object v) { return null; }\n" + 
 		"	       ^^^^^^^^^\n" + 
 		"The return type is incompatible with I<U,V>.foo(Object, V)\n" + 
-		"----------\n"
+		"----------\n",
+		JavacTestOptions.EclipseJustification.EclipseBug180789
 	);
 }
 // https://bugs.eclipse.org/bugs/show_bug.cgi?id=180789
diff --git a/org.eclipse.jdt.core.tests.compiler/src/org/eclipse/jdt/core/tests/compiler/regression/StaticImportTest.java b/org.eclipse.jdt.core.tests.compiler/src/org/eclipse/jdt/core/tests/compiler/regression/StaticImportTest.java
index c087ba5d5e..5ed73ddb82 100644
--- a/org.eclipse.jdt.core.tests.compiler/src/org/eclipse/jdt/core/tests/compiler/regression/StaticImportTest.java
+++ b/org.eclipse.jdt.core.tests.compiler/src/org/eclipse/jdt/core/tests/compiler/regression/StaticImportTest.java
@@ -142,7 +142,7 @@ public class StaticImportTest extends AbstractComparableTest {
 				"public class A implements I {}\n" +
 				"interface I { public static int C = 1; }\n"
 			},
-			"");
+			JavacTestOptions.JavacHasABug.JavacBugFixed_6_10);
 	}
 
 	public void test004() { // test static vs. instance
@@ -1391,7 +1391,7 @@ public class StaticImportTest extends AbstractComparableTest {
 				"interface I { int CONSTANT_I = 1; }\n" + 
 				"class B { public static int CONSTANT_B = 1; }",
 			}, 
-			"");
+			JavacTestOptions.JavacHasABug.JavacBugFixed_6_10);
 	}
 	//https://bugs.eclipse.org/bugs/show_bug.cgi?id=126564 - variation
 	public void test038() {
@@ -1520,7 +1520,8 @@ public class StaticImportTest extends AbstractComparableTest {
 	}
 	//https://bugs.eclipse.org/bugs/show_bug.cgi?id=134118
 	public void test041() {
-		this.runNegativeTest(
+		this.runConformTest(
+			true,
 			new String[] {
 				"Test.java",
 				"import static p.I.*;\n" + 
@@ -1540,7 +1541,10 @@ public class StaticImportTest extends AbstractComparableTest {
 			"	import static p.J.*;\n" + 
 			"	              ^^^\n" + 
 			"The import p.J is never used\n" + 
-			"----------\n"
+			"----------\n",
+			null,
+			null,
+			JavacTestOptions.Excuse.EclipseHasSomeMoreWarnings
 		);
 	}
 	//https://bugs.eclipse.org/bugs/show_bug.cgi?id=133737
@@ -2168,8 +2172,10 @@ public class StaticImportTest extends AbstractComparableTest {
 		}	
 	// https://bugs.eclipse.org/bugs/show_bug.cgi?id=183211 - variation
 	public void test061() {
-		this.runConformTest(
-			new String[] {
+		runConformTest(
+			// test directory preparation
+			true /* flush output directory */, 
+			new String[] { /* test files */
 				"p/X.java",
 				"package p;\n" + 
 				"import static q.A.a;\n" + 
@@ -2186,8 +2192,14 @@ public class StaticImportTest extends AbstractComparableTest {
 				"public class A implements B, I {\n" + 
 				"}\n",
 			},
-			"");		
-		}	
+			// compiler results
+			null /* do not check compiler log */,
+			// runtime results
+			"" /* expected output string */,
+			null /* do not check error string */,
+			// javac options
+			JavacTestOptions.EclipseJustification.EclipseBug183211 /* javac test options */);
+	}	
 	// https://bugs.eclipse.org/bugs/show_bug.cgi?id=183211 - variation
 	public void test062() {
 		this.runNegativeTest(
@@ -2256,7 +2268,8 @@ public class StaticImportTest extends AbstractComparableTest {
 			"	System.out.println(b);\n" + 
 			"	                   ^\n" + 
 			"The field b is ambiguous\n" + 
-			"----------\n"
+			"----------\n",
+			JavacTestOptions.EclipseJustification.EclipseBug183211b
 		);		
 	}
 	// https://bugs.eclipse.org/bugs/show_bug.cgi?id=183211 - variation
@@ -2280,7 +2293,8 @@ public class StaticImportTest extends AbstractComparableTest {
 			"	M m;\n" + 
 			"	^\n" + 
 			"The type M is ambiguous\n" + 
-			"----------\n"
+			"----------\n",
+			JavacTestOptions.EclipseJustification.EclipseBug183211
 		);
 	}
 	// https://bugs.eclipse.org/bugs/show_bug.cgi?id=230026