Bug 507571 - M3 Plugin JUnit tests give "Unable to index..."

Change the way file existence is checked. Rather than using
File.exists(), we rely on exceptions thrown during read and
the implicit existence check that occurs when testing fingerprints.
Remove the call to File.exists().

Change exception handling such that exceptions indicating missing
files are propagated reliably.

Update the invalid archive cache to record an extra state if a missing
file was the cause of the exception.

Change the way nonexistent files are stored in the database.
Previously, they were either left with a stale state from the
last successful indexing operation or they were marked as corrupt
files. Now they are stored in the same form as empty files (they
can be disambiguated from empty files by examining the fingerprint).

Omit logging any exceptions in response to missing files.

Change-Id: I244bd781cd3ef77b661041302bbef32253330476
Signed-off-by: Stefan Xenos <sxenos@gmail.com>

6 files changed, 84 insertions, 40 deletions

diff --git a/org.eclipse.jdt.core.tests.builder/src/org/eclipse/jdt/core/tests/builder/TestingEnvironment.java b/org.eclipse.jdt.core.tests.builder/src/org/eclipse/jdt/core/tests/builder/TestingEnvironment.java
index 40e1531d33..e6d2a97da9 100644
--- a/org.eclipse.jdt.core.tests.builder/src/org/eclipse/jdt/core/tests/builder/TestingEnvironment.java
+++ b/org.eclipse.jdt.core.tests.builder/src/org/eclipse/jdt/core/tests/builder/TestingEnvironment.java
@@ -21,6 +21,8 @@ import org.eclipse.jdt.core.tests.util.Util;
 import org.eclipse.jdt.internal.compiler.impl.CompilerOptions;
 import org.eclipse.jdt.internal.core.ClasspathEntry;
 import org.eclipse.jdt.internal.core.JavaProject;
+import org.eclipse.jdt.internal.core.nd.indexer.Indexer;
+
 import java.io.*;
 import java.util.*;
 
@@ -1093,6 +1095,7 @@ public void cleanBuild(String projectName) {
 		do {
 			try {
 				Job.getJobManager().join(ResourcesPlugin.FAMILY_AUTO_BUILD, null);
+				Indexer.getInstance().waitForIndex(null);
 				wasInterrupted = false;
 			} catch (OperationCanceledException e) {
 				handle(e);
@@ -1108,6 +1111,7 @@ public void cleanBuild(String projectName) {
 		do {
 			try {
 				Job.getJobManager().join(ResourcesPlugin.FAMILY_MANUAL_REFRESH, null);
+				Indexer.getInstance().waitForIndex(null);
 				wasInterrupted = false;
 			} catch (OperationCanceledException e) {
 				handle(e);
diff --git a/org.eclipse.jdt.core/model/org/eclipse/jdt/internal/core/JavaModelManager.java b/org.eclipse.jdt.core/model/org/eclipse/jdt/internal/core/JavaModelManager.java
index b898a64f86..c250a8c72f 100644
--- a/org.eclipse.jdt.core/model/org/eclipse/jdt/internal/core/JavaModelManager.java
+++ b/org.eclipse.jdt.core/model/org/eclipse/jdt/internal/core/JavaModelManager.java
@@ -27,6 +27,7 @@ import java.io.DataInputStream;
 import java.io.DataOutputStream;
 import java.io.File;
 import java.io.FileInputStream;
+import java.io.FileNotFoundException;
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.StringReader;
@@ -173,7 +174,7 @@ public class JavaModelManager implements ISaveParticipant, IContentTypeChangeLis
 	private static final String ASSUMED_EXTERNAL_FILES_CACHE = "assumedExternalFilesCache";  //$NON-NLS-1$
 
 	public static enum ArchiveValidity {
-		BAD_FORMAT, UNABLE_TO_READ, VALID;
+		BAD_FORMAT, UNABLE_TO_READ, FILE_NOT_FOUND, VALID;
 
 		public boolean isValid() {
 			return this == VALID;
@@ -2767,7 +2768,10 @@ public class JavaModelManager implements ISaveParticipant, IContentTypeChangeLis
 	 * zip/jar, or it must be an absolute workspace relative path if
 	 * representing a zip/jar inside the workspace.
 	 *
-	 * @exception CoreException If unable to create/open the ZipFile
+	 * @exception CoreException If unable to create/open the ZipFile. The
+	 * cause will be a {@link ZipException} if the file was corrupt, a
+	 * {@link FileNotFoundException} if the file does not exist, or a
+	 * {@link IOException} if we were unable to read the file.
 	 */
 	public ZipFile getZipFile(IPath path) throws CoreException {
 		return getZipFile(path, true);
@@ -2782,7 +2786,7 @@ public class JavaModelManager implements ISaveParticipant, IContentTypeChangeLis
 	 */
 	public static boolean throwIoExceptionsInGetZipFile = false;
 
-	private ZipFile getZipFile(IPath path, boolean checkInvalidArchiveCache) throws CoreException {
+	public ZipFile getZipFile(IPath path, boolean checkInvalidArchiveCache) throws CoreException {
 		if (checkInvalidArchiveCache) {
 			throwExceptionIfArchiveInvalid(path);
 		}
@@ -2807,7 +2811,15 @@ public class JavaModelManager implements ISaveParticipant, IContentTypeChangeLis
 			}
 			return zipFile;
 		} catch (IOException e) {
-			ArchiveValidity reason = (e instanceof ZipException) ? ArchiveValidity.BAD_FORMAT : ArchiveValidity.UNABLE_TO_READ;
+			ArchiveValidity reason; 
+			
+			if (e instanceof ZipException) {
+				reason = ArchiveValidity.BAD_FORMAT;
+			} else if (e instanceof FileNotFoundException) {
+				reason = ArchiveValidity.FILE_NOT_FOUND;
+			} else {
+				reason = ArchiveValidity.UNABLE_TO_READ;
+			}
 			addInvalidArchive(path, reason);
 			throw new CoreException(new Status(IStatus.ERROR, JavaCore.PLUGIN_ID, -1, Messages.status_IOException, e));
 		}
@@ -2835,13 +2847,14 @@ public class JavaModelManager implements ISaveParticipant, IContentTypeChangeLis
 
 	private void throwExceptionIfArchiveInvalid(IPath path) throws CoreException {
 		ArchiveValidity validity = getArchiveValidity(path);
-		if (!validity.isValid()) {
-			IOException reason;
-			if (validity == ArchiveValidity.BAD_FORMAT) {
-				reason = new ZipException();
-			} else {
-				reason = new IOException();
-			}
+		IOException reason;
+		switch (validity) {
+			case BAD_FORMAT: reason = new ZipException(); break;
+			case FILE_NOT_FOUND: reason = new FileNotFoundException(); break;
+			case UNABLE_TO_READ: reason = new IOException(); break;
+			default: reason = null;
+		}
+		if (reason != null) {
 			throw new CoreException(new Status(IStatus.ERROR, JavaCore.PLUGIN_ID, -1, Messages.status_IOException, reason));
 		}
 	}
diff --git a/org.eclipse.jdt.core/search/org/eclipse/jdt/internal/core/nd/indexer/ClassFileToIndexConverter.java b/org.eclipse.jdt.core/search/org/eclipse/jdt/internal/core/nd/indexer/ClassFileToIndexConverter.java
index afd740ee0b..d52ce330e7 100644
--- a/org.eclipse.jdt.core/search/org/eclipse/jdt/internal/core/nd/indexer/ClassFileToIndexConverter.java
+++ b/org.eclipse.jdt.core/search/org/eclipse/jdt/internal/core/nd/indexer/ClassFileToIndexConverter.java
@@ -16,9 +16,7 @@ import java.util.List;
 
 import org.eclipse.core.runtime.CoreException;
 import org.eclipse.core.runtime.IProgressMonitor;
-import org.eclipse.jdt.core.IClassFile;
 import org.eclipse.jdt.core.compiler.CharOperation;
-import org.eclipse.jdt.internal.compiler.classfmt.ClassFormatException;
 import org.eclipse.jdt.internal.compiler.codegen.AnnotationTargetTypeConstants;
 import org.eclipse.jdt.internal.compiler.env.ClassSignature;
 import org.eclipse.jdt.internal.compiler.env.EnumConstantSignature;
@@ -69,8 +67,6 @@ import org.eclipse.jdt.internal.core.nd.java.NdTypeInterface;
 import org.eclipse.jdt.internal.core.nd.java.NdTypeParameter;
 import org.eclipse.jdt.internal.core.nd.java.NdTypeSignature;
 import org.eclipse.jdt.internal.core.nd.java.NdVariable;
-import org.eclipse.jdt.internal.core.nd.java.model.BinaryTypeDescriptor;
-import org.eclipse.jdt.internal.core.nd.java.model.BinaryTypeFactory;
 import org.eclipse.jdt.internal.core.nd.util.CharArrayUtils;
 import org.eclipse.jdt.internal.core.util.CharArrayBuffer;
 import org.eclipse.jdt.internal.core.util.Util;
@@ -97,12 +93,6 @@ public final class ClassFileToIndexConverter {
 		return this.resource.getNd();
 	}
 
-	public static IBinaryType getTypeFromClassFile(IClassFile iClassFile, IProgressMonitor monitor)
-			throws CoreException, ClassFormatException {
-		BinaryTypeDescriptor descriptor = BinaryTypeFactory.createDescriptor(iClassFile);
-		return BinaryTypeFactory.rawReadType(descriptor, true);
-	}
-
 	/**
 	 * Create a type info from the given class file in a jar and adds it to the given list of infos.
 	 *
diff --git a/org.eclipse.jdt.core/search/org/eclipse/jdt/internal/core/nd/indexer/Indexer.java b/org.eclipse.jdt.core/search/org/eclipse/jdt/internal/core/nd/indexer/Indexer.java
index 7ca27245d9..23807803c2 100644
--- a/org.eclipse.jdt.core/search/org/eclipse/jdt/internal/core/nd/indexer/Indexer.java
+++ b/org.eclipse.jdt.core/search/org/eclipse/jdt/internal/core/nd/indexer/Indexer.java
@@ -10,7 +10,7 @@
  *******************************************************************************/
 package org.eclipse.jdt.internal.core.nd.indexer;
 
-import java.io.File;
+import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.util.ArrayDeque;
 import java.util.ArrayList;
@@ -567,14 +567,6 @@ public final class Indexer {
 		String pathString = thePath.toString();
 		JavaIndex javaIndex = JavaIndex.getIndex(this.nd);
 
-		File theFile = thePath.toFile();
-		if (!(theFile.exists() && theFile.isFile())) {
-			if (DEBUG) {
-				Package.log("the file " + pathString + " does not exist", null); //$NON-NLS-1$ //$NON-NLS-2$
-			}
-			return 0;
-		}
-
 		NdResourceFile resourceFile;
 
 		this.nd.acquireWriteLock(subMonitor.split(5));
@@ -596,9 +588,11 @@ public final class Indexer {
 		if (DEBUG) {
 			Package.logInfo("rescanning " + thePath.toString() + ", " + fingerprint); //$NON-NLS-1$ //$NON-NLS-2$
 		}
-		int result;
+		int result = 0;
 		try {
-			result = addElement(resourceFile, element, subMonitor.split(50));
+			if (fingerprint.fileExists()) {
+				result = addElement(resourceFile, element, subMonitor.split(50));
+			}
 		} catch (JavaModelException e) {
 			if (DEBUG) {
 				Package.log("the file " + pathString + " cannot be indexed due to a recoverable error", null); //$NON-NLS-1$ //$NON-NLS-2$
@@ -618,6 +612,12 @@ public final class Indexer {
 				Package.log("A RuntimeException occurred while indexing " + pathString, e); //$NON-NLS-1$
 			}
 			throw e;
+		} catch (FileNotFoundException e) {
+			fingerprint = FileFingerprint.getEmpty();
+		}
+
+		if (DEBUG && !fingerprint.fileExists()) {
+			Package.log("the file " + pathString + " was not indexed because it does not exist", null); //$NON-NLS-1$ //$NON-NLS-2$
 		}
 
 		List<NdResourceFile> allResourcesWithThisPath = Collections.emptyList();
@@ -655,9 +655,10 @@ public final class Indexer {
 
 	/**
 	 * Adds an archive to the index, under the given NdResourceFile.
+	 * @throws FileNotFoundException if the file does not exist
 	 */
 	private int addElement(NdResourceFile resourceFile, IJavaElement element, IProgressMonitor monitor)
-			throws JavaModelException {
+			throws JavaModelException, FileNotFoundException {
 		SubMonitor subMonitor = SubMonitor.convert(monitor);
 
 		if (element instanceof JarPackageFragmentRoot) {
@@ -711,6 +712,8 @@ public final class Indexer {
 			} catch (ZipException e) {
 				Package.log("The zip file " + jarRoot.getPath() + " was corrupt", e);  //$NON-NLS-1$//$NON-NLS-2$
 				// Indicates a corrupt zip file. Treat this like an empty zip file.
+			} catch (FileNotFoundException e) {
+				throw e;
 			} catch (IOException ioException) {
 				throw new JavaModelException(ioException, IJavaModelStatusConstants.IO_EXCEPTION);
 			} catch (CoreException coreException) {
@@ -729,7 +732,7 @@ public final class Indexer {
 
 			boolean indexed = false;
 			try {
-				ClassFileReader classFileReader = BinaryTypeFactory.rawReadType(descriptor, true);
+				ClassFileReader classFileReader = BinaryTypeFactory.rawReadTypeTestForExists(descriptor, true, false);
 				if (classFileReader != null) {
 					indexed = addClassToIndex(resourceFile, descriptor.fieldDescriptor, descriptor.indexPath,
 							classFileReader, iterationMonitor);
diff --git a/org.eclipse.jdt.core/search/org/eclipse/jdt/internal/core/nd/java/FileFingerprint.java b/org.eclipse.jdt.core/search/org/eclipse/jdt/internal/core/nd/java/FileFingerprint.java
index f1b02622f9..b5d6062477 100644
--- a/org.eclipse.jdt.core/search/org/eclipse/jdt/internal/core/nd/java/FileFingerprint.java
+++ b/org.eclipse.jdt.core/search/org/eclipse/jdt/internal/core/nd/java/FileFingerprint.java
@@ -139,6 +139,15 @@ public class FileFingerprint {
 	}
 
 	/**
+	 * Returns true iff the file existed at the time the fingerprint was computed.
+	 * 
+	 * @return true iff the file existed at the time the fingerprint was computed.
+	 */
+	public boolean fileExists() {
+		return !equals(EMPTY);
+	}
+
+	/**
 	 * Compares the given File with the receiver. If the fingerprint matches (ie: the file
 	 */
 	public FingerprintTestResult test(IPath path, IProgressMonitor monitor) throws CoreException {
diff --git a/org.eclipse.jdt.core/search/org/eclipse/jdt/internal/core/nd/java/model/BinaryTypeFactory.java b/org.eclipse.jdt.core/search/org/eclipse/jdt/internal/core/nd/java/model/BinaryTypeFactory.java
index e9ce570adb..631c3847ab 100644
--- a/org.eclipse.jdt.core/search/org/eclipse/jdt/internal/core/nd/java/model/BinaryTypeFactory.java
+++ b/org.eclipse.jdt.core/search/org/eclipse/jdt/internal/core/nd/java/model/BinaryTypeFactory.java
@@ -10,15 +10,19 @@
  *******************************************************************************/
 package org.eclipse.jdt.internal.core.nd.java.model;
 
+import java.io.FileNotFoundException;
 import java.io.IOException;
+import java.io.InputStream;
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipFile;
 
 import org.eclipse.core.resources.IFile;
+import org.eclipse.core.resources.IResourceStatus;
 import org.eclipse.core.resources.ResourcesPlugin;
 import org.eclipse.core.runtime.CoreException;
 import org.eclipse.core.runtime.IPath;
 import org.eclipse.core.runtime.IProgressMonitor;
+import org.eclipse.core.runtime.IStatus;
 import org.eclipse.core.runtime.Path;
 import org.eclipse.jdt.core.IClassFile;
 import org.eclipse.jdt.core.IJavaElement;
@@ -112,9 +116,8 @@ public class BinaryTypeFactory {
 	 * no such type exists.
 	 * @throws ClassFormatException 
 	 */
-	public static IBinaryType readType(BinaryTypeDescriptor descriptor, 
-			IProgressMonitor monitor) throws JavaModelException, ClassFormatException {
-		
+	public static IBinaryType readType(BinaryTypeDescriptor descriptor, IProgressMonitor monitor) throws JavaModelException, ClassFormatException {
+
 		if (JavaIndex.isEnabled()) {
 			try {
 				return readFromIndex(JavaIndex.getIndex(), descriptor, monitor);
@@ -126,6 +129,14 @@ public class BinaryTypeFactory {
 		return rawReadType(descriptor, true);
 	}
 
+	public static ClassFileReader rawReadType(BinaryTypeDescriptor descriptor, boolean fullyInitialize) throws JavaModelException, ClassFormatException {
+		try {
+			return rawReadTypeTestForExists(descriptor, fullyInitialize, true);
+		} catch (FileNotFoundException e) {
+			throw new JavaModelException(e, IJavaModelStatusConstants.IO_EXCEPTION);
+		}
+	}
+
 	/**
 	 * Read the class file from disk, circumventing the index's cache. This should only be used by callers
 	 * that need to read information from the class file which aren't present in the index (such as method bodies).
@@ -133,15 +144,18 @@ public class BinaryTypeFactory {
 	 * @return the newly-created ClassFileReader or null if the given class file does not exist.
 	 * @throws ClassFormatException if the class file existed but was corrupt
 	 * @throws JavaModelException if unable to read the class file due to a transient failure
+	 * @throws FileNotFoundException if the file does not exist
 	 */
-	public static ClassFileReader rawReadType(BinaryTypeDescriptor descriptor, boolean fullyInitialize) throws JavaModelException, ClassFormatException {
+	public static ClassFileReader rawReadTypeTestForExists(BinaryTypeDescriptor descriptor, boolean fullyInitialize,
+			boolean useInvalidArchiveCache) throws JavaModelException, ClassFormatException, FileNotFoundException {
 		if (descriptor == null) {
 			return null;
 		}
 		if (descriptor.isInJarFile()) {
 			ZipFile zip = null;
 			try {
-				zip = JavaModelManager.getJavaModelManager().getZipFile(new Path(new String(descriptor.workspacePath)));
+				zip = JavaModelManager.getJavaModelManager().getZipFile(new Path(new String(descriptor.workspacePath)),
+						useInvalidArchiveCache);
 				char[] entryNameCharArray = CharArrayUtils.concat(
 						JavaNames.fieldDescriptorToBinaryName(descriptor.fieldDescriptor), SuffixConstants.SUFFIX_class);
 				String entryName = new String(entryNameCharArray);
@@ -162,7 +176,18 @@ public class BinaryTypeFactory {
 			}
 		} else {
 			IFile file = ResourcesPlugin.getWorkspace().getRoot().getFile(new Path(new String(descriptor.workspacePath)));
-			byte[] contents = Util.getResourceContentsAsByteArray(file);
+			byte[] contents;
+			try (InputStream stream = file.getContents(true)) {
+				contents = org.eclipse.jdt.internal.compiler.util.Util.getInputStreamAsByteArray(stream, -1);
+			} catch (CoreException e) {
+				IStatus status = e.getStatus();
+				if (status.getCode() == IResourceStatus.RESOURCE_NOT_FOUND) {
+					throw new FileNotFoundException();
+				}
+				throw new JavaModelException(e);
+			} catch (IOException e) {
+				throw new JavaModelException(e, IJavaModelStatusConstants.IO_EXCEPTION);
+			}
 			return new ClassFileReader(contents, file.getFullPath().toString().toCharArray(), fullyInitialize);
 		}
 		return null;