Bug 535548 - Infinite recursion loading function whose return type refers to a parameter from the index

Change-Id: Ibff680983cc8ac0bd9d33d8f2247a9f0ca2fe2f3

7 files changed, 62 insertions, 21 deletions

diff --git a/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/internal/index/tests/IndexCPPTemplateResolutionTest.java b/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/internal/index/tests/IndexCPPTemplateResolutionTest.java
index 5fedc936e13..1011404ca27 100644
--- a/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/internal/index/tests/IndexCPPTemplateResolutionTest.java
+++ b/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/internal/index/tests/IndexCPPTemplateResolutionTest.java
@@ -3219,4 +3219,15 @@ public class IndexCPPTemplateResolutionTest extends IndexBindingResolutionTestBa
 	public void testClassCastException_533216() throws Exception {
 		checkBindings();
 	}
+	
+	//	template <int N>
+	//	struct array {};
+	//
+	//	template <class... Ts>
+	//	auto make_array(Ts... ts) -> array<sizeof...(ts)>;
+
+	//	auto x = make_array(2);
+	public void testRecursion_535548() throws Exception {
+		checkBindings();
+	}
 }
diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/dom/parser/ProblemFunctionType.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/dom/parser/ProblemFunctionType.java
index c9bab562820..99fc4143f98 100644
--- a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/dom/parser/ProblemFunctionType.java
+++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/dom/parser/ProblemFunctionType.java
@@ -22,7 +22,8 @@ import org.eclipse.core.runtime.CoreException;
  */
 public class ProblemFunctionType extends ProblemType implements ICPPFunctionType {
 	@SuppressWarnings("hiding")
-	public static final IType RECURSION_IN_LOOKUP = new ProblemFunctionType(BINDING_RECURSION_IN_LOOKUP);
+	public static final ICPPFunctionType RECURSION_IN_LOOKUP = new ProblemFunctionType(BINDING_RECURSION_IN_LOOKUP);
+	public static final ICPPFunctionType NOT_PERSISTED = new ProblemFunctionType(TYPE_NOT_PERSISTED);
 
 	public ProblemFunctionType(int id) {
 		super(id);
diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/dom/parser/ProblemType.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/dom/parser/ProblemType.java
index eea497c10b6..f26f96c4ab5 100644
--- a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/dom/parser/ProblemType.java
+++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/dom/parser/ProblemType.java
@@ -26,6 +26,7 @@ public class ProblemType implements IProblemType, ISerializableType {
 	public static final IType CANNOT_DEDUCE_DECLTYPE_AUTO_TYPE = new ProblemType(TYPE_CANNOT_DEDUCE_DECLTYPE_AUTO_TYPE);
 	public static final IType ENUMERATION_EXPECTED = new ProblemType(TYPE_ENUMERATION_EXPECTED);
 	public static final IType NO_NAME = new ProblemType(TYPE_NO_NAME);
+	public static final IType NOT_PERSISTED = new ProblemType(TYPE_NOT_PERSISTED);
 	public static final IType RECURSION_IN_LOOKUP = new ProblemType(BINDING_RECURSION_IN_LOOKUP);
 	public static final IType UNKNOWN_FOR_EXPRESSION = new ProblemType(TYPE_UNKNOWN_FOR_EXPRESSION);
 	public static final IType UNRESOLVED_NAME = new ProblemType(TYPE_UNRESOLVED_NAME);
diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/dom/parser/cpp/CPPDeferredClassInstance.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/dom/parser/cpp/CPPDeferredClassInstance.java
index 040ff241881..a1c7f75fb4d 100644
--- a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/dom/parser/cpp/CPPDeferredClassInstance.java
+++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/dom/parser/cpp/CPPDeferredClassInstance.java
@@ -230,7 +230,7 @@ public class CPPDeferredClassInstance extends CPPUnknownBinding implements ICPPD
 	}
 
 	public static ICPPDeferredClassInstance unmarshal(IIndexFragment fragment, short firstBytes,
-			ITypeMarshalBuffer buffer) throws CoreException {
+		ITypeMarshalBuffer buffer) throws CoreException {
 		IBinding template= buffer.unmarshalBinding();
 		int argcount= buffer.getInt();
 		ICPPTemplateArgument[] args = new ICPPTemplateArgument[argcount];
diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/dom/parser/cpp/semantics/CPPTemplates.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/dom/parser/cpp/semantics/CPPTemplates.java
index d3df642946e..56f7c837cd7 100644
--- a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/dom/parser/cpp/semantics/CPPTemplates.java
+++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/dom/parser/cpp/semantics/CPPTemplates.java
@@ -167,7 +167,6 @@ import org.eclipse.cdt.internal.core.dom.parser.cpp.CPPMethodSpecialization;
 import org.eclipse.cdt.internal.core.dom.parser.cpp.CPPMethodTemplateSpecialization;
 import org.eclipse.cdt.internal.core.dom.parser.cpp.CPPParameterPackType;
 import org.eclipse.cdt.internal.core.dom.parser.cpp.CPPParameterSpecialization;
-import org.eclipse.cdt.internal.core.dom.parser.cpp.CPPPlaceholderType;
 import org.eclipse.cdt.internal.core.dom.parser.cpp.CPPPointerToMemberType;
 import org.eclipse.cdt.internal.core.dom.parser.cpp.CPPPointerType;
 import org.eclipse.cdt.internal.core.dom.parser.cpp.CPPTemplateDefinition;
diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/pdom/dom/PDOMLinkage.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/pdom/dom/PDOMLinkage.java
index da1e67989f1..cd8f807f22c 100644
--- a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/pdom/dom/PDOMLinkage.java
+++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/pdom/dom/PDOMLinkage.java
@@ -17,6 +17,8 @@ package org.eclipse.cdt.internal.core.pdom.dom;
 
 import java.lang.ref.Reference;
 import java.lang.ref.SoftReference;
+import java.util.HashSet;
+import java.util.Set;
 
 import org.eclipse.cdt.core.dom.IPDOMVisitor;
 import org.eclipse.cdt.core.dom.ast.IASTDeclSpecifier;
@@ -41,6 +43,7 @@ import org.eclipse.cdt.core.parser.util.CharArrayMap;
 import org.eclipse.cdt.internal.core.dom.parser.ASTInternal;
 import org.eclipse.cdt.internal.core.dom.parser.ITypeMarshalBuffer;
 import org.eclipse.cdt.internal.core.dom.parser.ProblemBinding;
+import org.eclipse.cdt.internal.core.dom.parser.ProblemType;
 import org.eclipse.cdt.internal.core.dom.parser.cpp.ICPPEvaluation;
 import org.eclipse.cdt.internal.core.dom.parser.cpp.ICPPExecution;
 import org.eclipse.cdt.internal.core.index.IIndexBindingConstants;
@@ -77,6 +80,17 @@ public abstract class PDOMLinkage extends PDOMNamedNode implements IIndexLinkage
 	private final PDOM fPDOM;
 	private final Database fDatabase;
 
+	/**
+	 * The set of types currently being loaded from the index on each thread, represented as record numbers.
+	 * This is used to guard against infinite recursion while loading types.
+	 */
+	private static final ThreadLocal<Set<Long>> fLoadTypeInProgress = new ThreadLocal<Set<Long>>() {
+		@Override
+		protected Set<Long> initialValue() {
+			return new HashSet<>();
+		}
+	};
+	
 	public PDOMLinkage(PDOM pdom, long record) {
 		super(null, record);
 		fPDOM= pdom;
@@ -550,23 +564,31 @@ public abstract class PDOMLinkage extends PDOMNamedNode implements IIndexLinkage
 	}
 
 	public IType loadType(long offset) throws CoreException {
-		final Database db= getDB();
-		final byte firstByte= db.getByte(offset);
-		byte[] data= null;
-		switch (firstByte) {
-		case TypeMarshalBuffer.INDIRECT_TYPE:
-			data = loadLinkedSerializedData(db, offset + 1);
-			break;
-		case TypeMarshalBuffer.UNSTORABLE_TYPE:
-			return TypeMarshalBuffer.UNSTORABLE_TYPE_PROBLEM;
-		case TypeMarshalBuffer.NULL_TYPE:
-			return null;
-		default:
-			data= new byte[Database.TYPE_SIZE];
-			db.getBytes(offset, data);
-			break;
+		Set<Long> recursionProtectionSet = fLoadTypeInProgress.get();
+		if (!recursionProtectionSet.add(offset)) {
+			return ProblemType.NOT_PERSISTED;
+		}
+		try {
+			final Database db= getDB();
+			final byte firstByte= db.getByte(offset);
+			byte[] data= null;
+			switch (firstByte) {
+			case TypeMarshalBuffer.INDIRECT_TYPE:
+				data = loadLinkedSerializedData(db, offset + 1);
+				break;
+			case TypeMarshalBuffer.UNSTORABLE_TYPE:
+				return TypeMarshalBuffer.UNSTORABLE_TYPE_PROBLEM;
+			case TypeMarshalBuffer.NULL_TYPE:
+				return null;
+			default:
+				data= new byte[Database.TYPE_SIZE];
+				db.getBytes(offset, data);
+				break;
+			}
+			return new TypeMarshalBuffer(this, data).unmarshalType();
+		} finally {
+			recursionProtectionSet.remove(offset);
 		}
-		return new TypeMarshalBuffer(this, data).unmarshalType();
 	}
 
 	public void storeBinding(long offset, IBinding binding) throws CoreException {
diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/pdom/dom/cpp/PDOMCPPFunction.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/pdom/dom/cpp/PDOMCPPFunction.java
index 5b8307b0aab..0e1de2678a5 100644
--- a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/pdom/dom/cpp/PDOMCPPFunction.java
+++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/pdom/dom/cpp/PDOMCPPFunction.java
@@ -333,10 +333,17 @@ class PDOMCPPFunction extends PDOMCPPBinding implements ICPPFunction, IPDOMOverl
 	public final ICPPFunctionType getType() {
 		if (fType == null) {
 			try {
-				fType = (ICPPFunctionType) getLinkage().loadType(record + FUNCTION_TYPE);
+				IType type = getLinkage().loadType(record + FUNCTION_TYPE);
+				if (type instanceof ICPPFunctionType) {
+					fType = (ICPPFunctionType) type;
+				} else {
+					// Something went wrong while loading the type and we didn't
+					// get a function type. Treat it similar to an exception.
+					fType = ProblemFunctionType.NOT_PERSISTED;
+				}
 			} catch (CoreException e) {
 				CCorePlugin.log(e);
-				fType = new ProblemFunctionType(ISemanticProblem.TYPE_NOT_PERSISTED);
+				fType = ProblemFunctionType.NOT_PERSISTED;
 			}
 		}
 		return fType;