Bug 532420 - Make Container Core Build indexing more efficient

- add new ICBuildConfiguration2 to keep API checks happy
- remove refreshScannerInfo method from ICBuildConfiguration and
  put it in ICBuildConfiguration2
- make CBuildConfiguration implement ICBuildConfiguration2
- update ContainerPropertyVolumesModel to use new Docker plug-ins
  using docker-client 8.9.2.
- fix MesonBuildConfiguration to use a job for each compile line being
  processed, then wait until all jobs are done before causing
  an reindex to occur (this will maximize parallelism when building
  in Containers)
- fix ContainerCommandLauncherFactory to save the project so we
  can exclude project directories when copying header files using
  the new Docker Tooling interfaces
- fix CoreBuildLaunchBarTracker to use ICBuildConfiguration2
  interface to make the call to refreshScannerInfo


Change-Id: I2138f5111614e7821e46c22731397a01035eac0a

6 files changed, 201 insertions, 46 deletions

diff --git a/core/org.eclipse.cdt.core/.settings/.api_filters b/core/org.eclipse.cdt.core/.settings/.api_filters
index e4899e4dd3a..5380e1fd78c 100644
--- a/core/org.eclipse.cdt.core/.settings/.api_filters
+++ b/core/org.eclipse.cdt.core/.settings/.api_filters
@@ -4,12 +4,6 @@
         <filter id="403767336">

             <message_arguments>

                 <message_argument value="org.eclipse.cdt.core.build.ICBuildConfiguration"/>

-                <message_argument value="TOOLCHAIN_ID"/>

-            </message_arguments>

-        </filter>

-        <filter id="403767336">

-            <message_arguments>

-                <message_argument value="org.eclipse.cdt.core.build.ICBuildConfiguration"/>

                 <message_argument value="TOOLCHAIN_TYPE"/>

             </message_arguments>

         </filter>

diff --git a/core/org.eclipse.cdt.core/src/org/eclipse/cdt/core/build/CBuildConfiguration.java b/core/org.eclipse.cdt.core/src/org/eclipse/cdt/core/build/CBuildConfiguration.java
index 11344bafcce..9977f9d254e 100644
--- a/core/org.eclipse.cdt.core/src/org/eclipse/cdt/core/build/CBuildConfiguration.java
+++ b/core/org.eclipse.cdt.core/src/org/eclipse/cdt/core/build/CBuildConfiguration.java
@@ -80,6 +80,7 @@ import org.eclipse.core.runtime.NullProgressMonitor;
 import org.eclipse.core.runtime.Platform;
 import org.eclipse.core.runtime.PlatformObject;
 import org.eclipse.core.runtime.Status;
+import org.eclipse.core.runtime.jobs.Job;
 import org.eclipse.core.runtime.preferences.InstanceScope;
 import org.osgi.service.prefs.BackingStoreException;
 import org.osgi.service.prefs.Preferences;
@@ -100,7 +101,8 @@ import com.google.gson.JsonParseException;
  * @since 6.0
  */
 public abstract class CBuildConfiguration extends PlatformObject
-		implements ICBuildConfiguration, IMarkerGenerator, IConsoleParser, IElementChangedListener {
+		implements ICBuildConfiguration, ICBuildConfiguration2, IMarkerGenerator, 
+		IConsoleParser, IElementChangedListener {
 
 	private static final String LAUNCH_MODE = "cdt.launchMode"; //$NON-NLS-1$
 
@@ -110,6 +112,8 @@ public abstract class CBuildConfiguration extends PlatformObject
 	private final IBuildConfiguration config;
 	private final IToolChain toolChain;
 	private String launchMode;
+	
+	private Object scannerInfoLock = new Object();
 
 	private final Map<IResource, List<IScannerInfoChangeListener>> scannerInfoListeners = new HashMap<>();
 	private ScannerInfoCache scannerInfoCache;
@@ -613,31 +617,33 @@ public abstract class CBuildConfiguration extends PlatformObject
 	/**
 	 * @since 6.1
 	 */
-	protected synchronized void loadScannerInfoCache() {
-		if (scannerInfoCache == null) {
-			File cacheFile = getScannerInfoCacheFile();
-			if (cacheFile.exists()) {
-				try (FileReader reader = new FileReader(cacheFile)) {
-					GsonBuilder gsonBuilder = new GsonBuilder();
-					gsonBuilder.registerTypeAdapter(IExtendedScannerInfo.class,
-							new IExtendedScannerInfoCreator());
-					Gson gson = gsonBuilder.create();
-					scannerInfoCache = gson.fromJson(reader, ScannerInfoCache.class);
-				} catch (IOException e) {
-					CCorePlugin.log(e);
+	protected void loadScannerInfoCache() {
+		synchronized (scannerInfoLock) {
+			if (scannerInfoCache == null) {
+				File cacheFile = getScannerInfoCacheFile();
+				if (cacheFile.exists()) {
+					try (FileReader reader = new FileReader(cacheFile)) {
+						GsonBuilder gsonBuilder = new GsonBuilder();
+						gsonBuilder.registerTypeAdapter(IExtendedScannerInfo.class,
+								new IExtendedScannerInfoCreator());
+						Gson gson = gsonBuilder.create();
+						scannerInfoCache = gson.fromJson(reader, ScannerInfoCache.class);
+					} catch (IOException e) {
+						CCorePlugin.log(e);
+						scannerInfoCache = new ScannerInfoCache();
+					}
+				} else {
 					scannerInfoCache = new ScannerInfoCache();
 				}
-			} else {
-				scannerInfoCache = new ScannerInfoCache();
+				scannerInfoCache.initCache();
 			}
-			scannerInfoCache.initCache();
 		}
 	}
 
 	/**
 	 * @since 6.1
 	 */
-	protected void saveScannerInfoCache() {
+	protected synchronized void saveScannerInfoCache() {
 		File cacheFile = getScannerInfoCacheFile();
 		if (!cacheFile.getParentFile().exists()) {
 			try {
@@ -650,7 +656,9 @@ public abstract class CBuildConfiguration extends PlatformObject
 
 		try (FileWriter writer = new FileWriter(getScannerInfoCacheFile())) {
 			Gson gson = new Gson();
-			gson.toJson(scannerInfoCache, writer);
+			synchronized (scannerInfoLock) {
+				gson.toJson(scannerInfoCache, writer);
+			}
 		} catch (IOException e) {
 			CCorePlugin.log(e);
 		}
@@ -694,7 +702,10 @@ public abstract class CBuildConfiguration extends PlatformObject
 	@Override
 	public IScannerInfo getScannerInformation(IResource resource) {
 		loadScannerInfoCache();
-		IExtendedScannerInfo info = scannerInfoCache.getScannerInfo(resource);
+		IExtendedScannerInfo info = null;
+		synchronized (scannerInfoLock) {
+			info = scannerInfoCache.getScannerInfo(resource);
+		}
 		if (info == null) {
 			ICElement celement = CCorePlugin.getDefault().getCoreModel().create(resource);
 			if (celement instanceof ITranslationUnit) {
@@ -702,7 +713,9 @@ public abstract class CBuildConfiguration extends PlatformObject
 					ITranslationUnit tu = (ITranslationUnit) celement;
 					info = getToolChain().getDefaultScannerInfo(getBuildConfiguration(),
 							getBaseScannerInfo(resource), tu.getLanguage(), getBuildDirectoryURI());
-					scannerInfoCache.addScannerInfo(DEFAULT_COMMAND, info, resource);
+					synchronized (scannerInfoLock) {
+						scannerInfoCache.addScannerInfo(DEFAULT_COMMAND, info, resource);
+					}
 					saveScannerInfoCache();
 				} catch (CoreException e) {
 					CCorePlugin.log(e.getStatus());
@@ -731,12 +744,14 @@ public abstract class CBuildConfiguration extends PlatformObject
 				IResource resource = delta.getElement().getResource();
 				if (resource.getProject().equals(getProject())) {
 					loadScannerInfoCache();
-					if (scannerInfoCache.hasResource(DEFAULT_COMMAND, resource)) {
-						scannerInfoCache.removeResource(resource);
-					} else {
-						// Clear the whole command and exit the delta
-						scannerInfoCache.removeCommand(DEFAULT_COMMAND);
-						return;
+					synchronized (scannerInfoLock) {
+						if (scannerInfoCache.hasResource(DEFAULT_COMMAND, resource)) {
+							scannerInfoCache.removeResource(resource);
+						} else {
+							// Clear the whole command and exit the delta
+							scannerInfoCache.removeCommand(DEFAULT_COMMAND);
+							return;
+						}
 					}
 				}
 			}
@@ -852,19 +867,27 @@ public abstract class CBuildConfiguration extends PlatformObject
 
 				for (IResource resource : resources) {
 					loadScannerInfoCache();
-					if (scannerInfoCache.hasCommand(commandStrings)) {
-						if (!scannerInfoCache.hasResource(commandStrings, resource)) {
-							scannerInfoCache.addResource(commandStrings, resource);
-							infoChanged = true;
+					boolean hasCommand = true;
+					synchronized (scannerInfoLock) {
+						if (scannerInfoCache.hasCommand(commandStrings)) {
+							if (!scannerInfoCache.hasResource(commandStrings, resource)) {
+								scannerInfoCache.addResource(commandStrings, resource);
+								infoChanged = true;
+							}
+						} else {
+							hasCommand = false;
 						}
-					} else {
+					}
+					if (!hasCommand) {
 						Path commandPath = findCommand(command.get(0));
 						if (commandPath != null) {
 							command.set(0, commandPath.toString());
 							IExtendedScannerInfo info = getToolChain().getScannerInfo(getBuildConfiguration(),
 									command, null, resource, getBuildDirectoryURI());
-							scannerInfoCache.addScannerInfo(commandStrings, info, resource);
-							infoChanged = true;
+							synchronized (scannerInfoLock) {
+								scannerInfoCache.addScannerInfo(commandStrings, info, resource);
+								infoChanged = true;
+							}
 						}
 					}
 				}
@@ -877,8 +900,126 @@ public abstract class CBuildConfiguration extends PlatformObject
 			return false;
 		}
 	}
+
+	private class ScannerInfoJob extends Job {
+		private IToolChain toolchain;
+		private List<String> command;
+		private List<String> commandStrings;
+		private IResource resource;
+		private URI buildDirectoryURI;
+		
+		public ScannerInfoJob(String msg, IToolChain toolchain, List<String> command, IResource resource,
+				URI buildDirectoryURI, List<String> commandStrings) {
+			super(msg);
+			this.toolchain = toolchain;
+			this.command = command;
+			this.commandStrings = commandStrings;
+			this.resource = resource;
+			this.buildDirectoryURI = buildDirectoryURI;
+		}
+		
+		@Override
+		protected IStatus run(IProgressMonitor monitor) {
+			IExtendedScannerInfo info = toolchain.getScannerInfo(getBuildConfiguration(),
+					command, null, resource, buildDirectoryURI);
+			synchronized (scannerInfoLock) {
+				scannerInfoCache.addScannerInfo(commandStrings, info, resource);
+				infoChanged = true;
+			}
+			return Status.OK_STATUS;
+		}
+	}
 	
 	/**
+	 * Process a compile line for Scanner info in a separate job
+	 * 
+	 * @param line - line to process
+	 * @param jobsArray - array of Jobs to keep track of open scanner info jobs
+	 * @return - true if line processed, false otherwise
+	 * 
+	 * @since 6.5
+	 */
+	protected boolean processLine(String line, List<Job> jobsArray) {
+		// Split line into args, taking into account quotes
+		List<String> command = stripArgs(line);
+		
+		// Make sure it's a compile command
+		String[] compileCommands = toolChain.getCompileCommands();
+		boolean found = false;
+		loop:
+		for (String arg : command) {
+			// TODO we should really ask the toolchain, not all args start with '-'
+			if (arg.startsWith("-")) { //$NON-NLS-1$
+				// option found, missed our command
+				return false;
+			}
+
+			for (String cc : compileCommands) {
+				if (arg.endsWith(cc)
+						&& (arg.equals(cc) || arg.endsWith("/" + cc) || arg.endsWith("\\" + cc))) { //$NON-NLS-1$ //$NON-NLS-2$
+					found = true;
+					break loop;
+				}
+			}
+
+
+			if (Platform.getOS().equals(Platform.OS_WIN32) && !arg.endsWith(".exe")) { //$NON-NLS-1$
+				// Try with exe
+				arg = arg + ".exe"; //$NON-NLS-1$
+				for (String cc : compileCommands) {
+					if (arg.endsWith(cc)
+							&& (arg.equals(cc) || arg.endsWith("/" + cc) || arg.endsWith("\\" + cc))) { //$NON-NLS-1$ //$NON-NLS-2$
+						found = true;
+						break loop;
+					}
+				}
+			}
+		}
+
+		if (!found) {
+			return false;
+		}
+
+		try {
+			IResource[] resources = toolChain.getResourcesFromCommand(command, getBuildDirectoryURI());
+			if (resources != null && resources.length > 0) {
+				List<String> commandStrings = toolChain.stripCommand(command, resources);
+
+				for (IResource resource : resources) {
+					loadScannerInfoCache();
+					boolean hasCommand = true;
+					synchronized (scannerInfoLock) {
+						if (scannerInfoCache.hasCommand(commandStrings)) {
+							if (!scannerInfoCache.hasResource(commandStrings, resource)) {
+								scannerInfoCache.addResource(commandStrings, resource);
+								infoChanged = true;
+							}
+						} else {
+							hasCommand = false;
+						}
+					}
+					if (!hasCommand) {
+						Path commandPath = findCommand(command.get(0));
+						if (commandPath != null) {
+							command.set(0, commandPath.toString());
+							Job job = new ScannerInfoJob(String.format(Messages.CBuildConfiguration_RunningScannerInfo, resource), 
+									getToolChain(), command, resource, getBuildDirectoryURI(), commandStrings);
+							job.schedule();
+							jobsArray.add(job);
+						}
+					}
+				}
+				return true;
+			} else {
+				return false;
+			}
+		} catch (CoreException e) {
+			CCorePlugin.log(e);
+			return false;
+		}
+	}
+
+	/**
 	 * @since 6.5
 	 */
 	@Override
diff --git a/core/org.eclipse.cdt.core/src/org/eclipse/cdt/core/build/ICBuildConfiguration.java b/core/org.eclipse.cdt.core/src/org/eclipse/cdt/core/build/ICBuildConfiguration.java
index 91d7823815e..1d295b31dec 100644
--- a/core/org.eclipse.cdt.core/src/org/eclipse/cdt/core/build/ICBuildConfiguration.java
+++ b/core/org.eclipse.cdt.core/src/org/eclipse/cdt/core/build/ICBuildConfiguration.java
@@ -126,13 +126,6 @@ public interface ICBuildConfiguration extends IAdaptable, IScannerInfoProvider {
 	void clean(IConsole console, IProgressMonitor monitor) throws CoreException;
 	
 	/**
-	 * Refresh the Scanner info
-	 * 
-	 * @since 6.5
-	 */
-	void refreshScannerInfo() throws CoreException;
-
-	/**
 	 * The binaries produced by the build.
 	 * 
 	 * @return binaries produced by the build.
diff --git a/core/org.eclipse.cdt.core/src/org/eclipse/cdt/core/build/ICBuildConfiguration2.java b/core/org.eclipse.cdt.core/src/org/eclipse/cdt/core/build/ICBuildConfiguration2.java
new file mode 100644
index 00000000000..87dfea69ba3
--- /dev/null
+++ b/core/org.eclipse.cdt.core/src/org/eclipse/cdt/core/build/ICBuildConfiguration2.java
@@ -0,0 +1,25 @@
+/*******************************************************************************
+ * Copyright (c) 2018 Red Hat Inc. and others.
+ * All rights reserved. This program and the accompanying materials
+ * are made available under the terms of the Eclipse Public License v1.0
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/epl-v10.html
+ * 
+ * Contributors:
+ * 		Red Hat Inc. - initial implementation
+ *******************************************************************************/
+package org.eclipse.cdt.core.build;
+
+import org.eclipse.core.runtime.CoreException;
+
+/**
+ * @since 6.5
+ */
+public interface ICBuildConfiguration2 {
+
+	/**
+	 * Refresh the Scanner info
+	 */
+	void refreshScannerInfo() throws CoreException;
+
+}
diff --git a/core/org.eclipse.cdt.core/src/org/eclipse/cdt/internal/core/build/Messages.java b/core/org.eclipse.cdt.core/src/org/eclipse/cdt/internal/core/build/Messages.java
index 616f15db4f1..ba5626fc452 100644
--- a/core/org.eclipse.cdt.core/src/org/eclipse/cdt/internal/core/build/Messages.java
+++ b/core/org.eclipse.cdt.core/src/org/eclipse/cdt/internal/core/build/Messages.java
@@ -14,6 +14,7 @@ public class Messages extends NLS {
 	public static String CBuildConfiguration_CreateJob;
 	public static String CBuildConfiguration_ToolchainMissing;
 	public static String CBuildConfiguration_Location;
+	public static String CBuildConfiguration_RunningScannerInfo;
 	public static String CBuilder_ExceptionWhileBuilding;
 	public static String CBuilder_ExceptionWhileBuilding2;
 	public static String CBuilder_NotConfiguredCorrectly;
diff --git a/core/org.eclipse.cdt.core/src/org/eclipse/cdt/internal/core/build/messages.properties b/core/org.eclipse.cdt.core/src/org/eclipse/cdt/internal/core/build/messages.properties
index 3313cbe5a6a..9a5d1a8926f 100644
--- a/core/org.eclipse.cdt.core/src/org/eclipse/cdt/internal/core/build/messages.properties
+++ b/core/org.eclipse.cdt.core/src/org/eclipse/cdt/internal/core/build/messages.properties
@@ -15,3 +15,4 @@ StandardBuildConfiguration_CommandNotFound=Error: build command '%s' not found
 CBuildConfiguration_CreateJob=Create Build Folder
 CBuildConfiguration_Location=line %d, external location: %s
 CBuildConfiguration_ToolchainMissing=Toolchain is missing for build configuration
+CBuildConfiguration_RunningScannerInfo=Calculating scanner info for %s