Bug 515668 - Introduce APIs to group MavenProject reads

Using ProjectBuilder.build(singlePom...) has a speed and memory
complexity of O(depth(project)) where depth(project) is the number of
parents (recursively). Iterating on this over N projects lead to a
pseudo-quadratic cost O(N*maxProjectDepth) where maxProjectDepth can be
up to N-1.

Instead, we introduce APIs that take advantage
ProjectBuilder.build(multiplePom...) which has a complexity of O(N).

For Apache Camel that is 700+ modules, these new APIs instantiate  700+
MavenProject instead of ~4000 (since many modules have a depth of 5/6)
like legacy API do.

Change-Id: Ica74542eb8db6833de3b796bfad8c07c2ae9b002
Signed-off-by: Mickael Istria <mistria@redhat.com>

3 files changed, 138 insertions, 45 deletions

diff --git a/org.eclipse.m2e.core/src/org/eclipse/m2e/core/embedder/IMaven.java b/org.eclipse.m2e.core/src/org/eclipse/m2e/core/embedder/IMaven.java
index 334c1d37..f2084c6a 100644
--- a/org.eclipse.m2e.core/src/org/eclipse/m2e/core/embedder/IMaven.java
+++ b/org.eclipse.m2e.core/src/org/eclipse/m2e/core/embedder/IMaven.java
@@ -14,7 +14,9 @@ package org.eclipse.m2e.core.embedder;
 import java.io.File;
 import java.io.InputStream;
 import java.io.OutputStream;
+import java.util.Collection;
 import java.util.List;
+import java.util.Map;
 
 import org.eclipse.core.runtime.CoreException;
 import org.eclipse.core.runtime.IProgressMonitor;
@@ -104,10 +106,19 @@ public interface IMaven {
 
   /**
    * @since 1.4
+   * @see {@link #readMavenProjects(File, ProjectBuildingRequest)} to group requests and improve performance (RAM and
+   *      CPU)
    */
   public MavenExecutionResult readMavenProject(File pomFile, ProjectBuildingRequest configuration) throws CoreException;
 
   /**
+   * @since 1.10
+   */
+  public Map<File, MavenExecutionResult> readMavenProjects(Collection<File> pomFiles,
+      ProjectBuildingRequest configuration)
+      throws CoreException;
+
+  /**
    * Makes MavenProject instances returned by #readProject methods suitable for caching and reuse with other
    * MavenSession instances.<br/>
    * Do note that MavenProject.getParentProject() cannot be used for detached MavenProject instances,
diff --git a/org.eclipse.m2e.core/src/org/eclipse/m2e/core/internal/embedder/MavenImpl.java b/org.eclipse.m2e.core/src/org/eclipse/m2e/core/internal/embedder/MavenImpl.java
index 344ff5ea..65f86bfb 100644
--- a/org.eclipse.m2e.core/src/org/eclipse/m2e/core/internal/embedder/MavenImpl.java
+++ b/org.eclipse.m2e.core/src/org/eclipse/m2e/core/internal/embedder/MavenImpl.java
@@ -24,9 +24,11 @@ import java.io.InputStream;
 import java.io.OutputStream;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.LinkedHashMap;
 import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Map;
@@ -643,6 +645,42 @@ public class MavenImpl implements IMaven, IMavenConfigurationChangeListener {
     return result;
   }
 
+  public Map<File, MavenExecutionResult> readMavenProjects(Collection<File> pomFiles,
+      ProjectBuildingRequest configuration)
+      throws CoreException {
+    long start = System.currentTimeMillis();
+
+    log.debug("Reading {} Maven project(s): {}", pomFiles.size(), pomFiles.toString()); //$NON-NLS-1$
+
+    List<ProjectBuildingResult> projectBuildingResults = null;
+    Map<File, MavenExecutionResult> result = new LinkedHashMap<>(pomFiles.size(), 1.f);
+    try {
+      configuration.setValidationLevel(ModelBuildingRequest.VALIDATION_LEVEL_MINIMAL);
+      projectBuildingResults = lookup(ProjectBuilder.class).build(new ArrayList<>(pomFiles),
+          false,
+          configuration);
+    } catch(ProjectBuildingException ex) {
+      if(ex.getResults() != null) {
+        projectBuildingResults = ex.getResults();
+      }
+    } finally {
+      log.debug("Read {} Maven project(s) in {} ms",pomFiles.size(),System.currentTimeMillis()-start); //$NON-NLS-1$
+    }
+    if(projectBuildingResults != null) {
+      for (ProjectBuildingResult projectBuildingResult : projectBuildingResults) {
+        MavenExecutionResult mavenExecutionResult = new DefaultMavenExecutionResult();
+        mavenExecutionResult.setProject(projectBuildingResult.getProject());
+        mavenExecutionResult.setDependencyResolutionResult(projectBuildingResult.getDependencyResolutionResult());
+        if(!projectBuildingResult.getProblems().isEmpty()) {
+          mavenExecutionResult
+              .addException(new ProjectBuildingException(Collections.singletonList(projectBuildingResult)));
+        }
+        result.put(projectBuildingResult.getPomFile(), mavenExecutionResult);
+      }
+    }
+    return result;
+  }
+
   /**
    * Makes MavenProject instances returned by #readProject methods suitable for caching and reuse with other
    * MavenSession instances.<br/>
diff --git a/org.eclipse.m2e.core/src/org/eclipse/m2e/core/internal/project/registry/ProjectRegistryManager.java b/org.eclipse.m2e.core/src/org/eclipse/m2e/core/internal/project/registry/ProjectRegistryManager.java
index 05e5444d..4c41e37d 100644
--- a/org.eclipse.m2e.core/src/org/eclipse/m2e/core/internal/project/registry/ProjectRegistryManager.java
+++ b/org.eclipse.m2e.core/src/org/eclipse/m2e/core/internal/project/registry/ProjectRegistryManager.java
@@ -25,10 +25,13 @@ import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.ListIterator;
 import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Objects;
 import java.util.Properties;
 import java.util.Set;
 import java.util.concurrent.Callable;
 import java.util.concurrent.ExecutionException;
+import java.util.stream.Collectors;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -38,9 +41,12 @@ import com.google.common.cache.CacheBuilder;
 import com.google.common.cache.RemovalCause;
 import com.google.common.cache.RemovalListener;
 import com.google.common.cache.RemovalNotification;
+import com.google.common.collect.LinkedHashMultimap;
+import com.google.common.collect.Multimap;
 
 import org.eclipse.core.resources.IFile;
 import org.eclipse.core.resources.IProject;
+import org.eclipse.core.resources.IResource;
 import org.eclipse.core.resources.ResourcesPlugin;
 import org.eclipse.core.runtime.CoreException;
 import org.eclipse.core.runtime.IPath;
@@ -189,9 +195,10 @@ public class ProjectRegistryManager {
     MavenProjectFacade projectFacade = projectRegistry.getProjectFacade(pom);
     if(projectFacade == null && load) {
       ResolverConfiguration configuration = ResolverConfigurationIO.readResolverConfiguration(pom.getProject());
-      MavenExecutionResult executionResult = readProjectWithDependencies(projectRegistry, pom, configuration, monitor);
+      MavenExecutionResult executionResult = readProjectsWithDependencies(projectRegistry,
+          Collections.singletonList(pom), configuration, monitor).values().iterator().next();
       MavenProject mavenProject = executionResult.getProject();
-      if(mavenProject != null) {
+      if(mavenProject != null && mavenProject.getArtifact() != null) {
         projectFacade = new MavenProjectFacade(this, pom, mavenProject, configuration);
       } else {
         List<Throwable> exceptions = executionResult.getExceptions();
@@ -393,7 +400,7 @@ public class ProjectRegistryManager {
           context.forcePomFiles(newState.getVersionedDependents(mavenArtifactImportCapability, true));
         }
 
-        newFacade = readMavenProjectFacade(pom, newState, monitor);
+        newFacade = readMavenProjectFacades(Collections.singletonList(pom), newState, monitor).get(pom);
       } else {
         // refresh children of deleted/closed parent
         if(oldFacade != null) {
@@ -468,9 +475,9 @@ public class ProjectRegistryManager {
       }
       if(newFacade != null) {
         MavenProject mavenProject = getMavenProject(newFacade);
-        if(mavenProject == null) {
+        if(mavenProject == null || mavenProject.getArtifact() != null) {
           // facade from workspace state that has not been refreshed yet 
-          newFacade = readMavenProjectFacade(pom, newState, monitor);
+          newFacade = readMavenProjectFacades(Collections.singletonList(pom), newState, monitor).get(pom);
         } else {
           // recreate facade instance to trigger project changed event
           // this is only necessary for facades that are refreshed because their dependencies changed
@@ -684,36 +691,58 @@ public class ProjectRegistryManager {
     return new DefaultMavenDependencyResolver(this, markerManager);
   }
 
-  private MavenProjectFacade readMavenProjectFacade(final IFile pom, final MutableProjectRegistry state,
-      final IProgressMonitor monitor) throws CoreException {
-    markerManager.deleteMarkers(pom, IMavenConstants.MARKER_POM_LOADING_ID);
-
-    final ResolverConfiguration resolverConfiguration = ResolverConfigurationIO
-        .readResolverConfiguration(pom.getProject());
-
-    return execute(state, pom, resolverConfiguration, (executionContext, pm) -> {
-        MavenProject mavenProject = null;
-        MavenExecutionResult mavenResult = null;
-        if(pom.isAccessible()) {
-        mavenResult = getMaven().readMavenProject(pom.getLocation().toFile(),
-            executionContext.newProjectBuildingRequest());
-          mavenProject = mavenResult.getProject();
-        }
-
-        MarkerUtils.addEditorHintMarkers(markerManager, pom, mavenProject, IMavenConstants.MARKER_POM_LOADING_ID);
-        markerManager.addMarkers(pom, IMavenConstants.MARKER_POM_LOADING_ID, mavenResult);
-        if(mavenProject == null) {
-          return null;
-        }
+  private Map<IFile, MavenProjectFacade> readMavenProjectFacades(final Collection<IFile> poms,
+      final MutableProjectRegistry state, final IProgressMonitor monitor)
+      throws CoreException {
+    for(IFile pom : poms) {
+      markerManager.deleteMarkers(pom, IMavenConstants.MARKER_POM_LOADING_ID);
+    }
 
-        // create and return new project facade
-        MavenProjectFacade mavenProjectFacade = new MavenProjectFacade(ProjectRegistryManager.this, pom, mavenProject,
-            resolverConfiguration);
+    final Map<IFile, ResolverConfiguration> resolverConfigurations = new HashMap<>(poms.size(), 1.f);
+    final Multimap<ResolverConfiguration, IFile> groupsToImport = LinkedHashMultimap.create();
+    for(IFile pom : poms) {
+      if(monitor.isCanceled()) {
+        return null;
+      }
+      ResolverConfiguration resolverConfiguration = ResolverConfigurationIO.readResolverConfiguration(pom.getProject());
+      resolverConfigurations.put(pom, resolverConfiguration);
+      groupsToImport.put(resolverConfiguration, pom);
+    }
 
-        putMavenProject(mavenProjectFacade, mavenProject); // maintain maven project cache
+    Map<IFile, MavenProjectFacade> result = new HashMap<>(poms.size(), 1.f);
+    SubMonitor subMonitor = SubMonitor.convert(monitor, poms.size());
+    for(Entry<ResolverConfiguration, Collection<IFile>> entry : groupsToImport.asMap().entrySet()) {
+      ResolverConfiguration resolverConfiguration = entry.getKey();
+      Collection<IFile> pomFiles = entry.getValue();
+      result.putAll(execute(state, poms.size() == 1 ? pomFiles.iterator().next() : null, resolverConfiguration,
+          (executionContext, pm) -> {
+            Map<File, MavenExecutionResult> mavenResults = getMaven().readMavenProjects(pomFiles.stream()
+                .filter(IFile::isAccessible).map(pom -> pom.getLocation().toFile()).collect(Collectors.toList()),
+                executionContext.newProjectBuildingRequest());
+
+            Map<IFile, MavenProjectFacade> facades = new HashMap<>(mavenResults.size(), 1.f);
+            for (IFile pom : pomFiles) {
+              if (!pom.isAccessible()) {
+            	  continue;
+              }
+              MavenExecutionResult mavenResult = mavenResults.get(pom.getLocation().toFile());
+              MavenProject mavenProject = mavenResult.getProject();
+              MarkerUtils.addEditorHintMarkers(markerManager, pom, mavenProject,
+                      IMavenConstants.MARKER_POM_LOADING_ID);
+              markerManager.addMarkers(pom, IMavenConstants.MARKER_POM_LOADING_ID, mavenResult);
+              if(mavenProject != null && mavenProject.getArtifact() != null) {
+                MavenProjectFacade mavenProjectFacade = new MavenProjectFacade(ProjectRegistryManager.this, pom, mavenProject,
+                          resolverConfiguration);
+                putMavenProject(mavenProjectFacade, mavenProject); // maintain maven project cache
+                facades.put(pom, mavenProjectFacade);
+              }
+            }
 
-        return mavenProjectFacade;
-    }, monitor);
+            return facades;
+          }, subMonitor.split(pomFiles.size())
+      ));
+    }
+    return result;
   }
 
       /*package*/Map<String, List<MojoExecution>> calculateExecutionPlans(IFile pom, MavenProject mavenProject,
@@ -785,9 +814,14 @@ public class ProjectRegistryManager {
 
   MavenProject readProjectWithDependencies(IFile pomFile, ResolverConfiguration resolverConfiguration,
       IProgressMonitor monitor) throws CoreException {
-    MavenExecutionResult result = readProjectWithDependencies(projectRegistry, pomFile, resolverConfiguration, monitor);
+    Map<File, MavenExecutionResult> results = readProjectsWithDependencies(projectRegistry,
+        Collections.singletonList(pomFile), resolverConfiguration, monitor);
+    if(results.size() != 1) {
+      throw new IllegalStateException("Results should contain one entry.");
+    }
+    MavenExecutionResult result = results.values().iterator().next();
     MavenProject mavenProject = result.getProject();
-    if(mavenProject != null) {
+    if(mavenProject != null && result.getExceptions().isEmpty()) {
       return mavenProject;
     }
     MultiStatus status = new MultiStatus(IMavenConstants.PLUGIN_ID, 0, Messages.MavenProjectFacade_error, null);
@@ -798,21 +832,24 @@ public class ProjectRegistryManager {
     throw new CoreException(status);
   }
 
-  private MavenExecutionResult readProjectWithDependencies(IProjectRegistry state, final IFile pomFile,
-      ResolverConfiguration resolverConfiguration, final IProgressMonitor monitor) {
-
+  Map<File, MavenExecutionResult> readProjectsWithDependencies(IProjectRegistry state, Collection<IFile> pomFiles,
+      ResolverConfiguration resolverConfiguration, IProgressMonitor monitor) {
     try {
-      return execute(state, pomFile, resolverConfiguration, (context, pm) -> {
-          ProjectBuildingRequest configuration = context.newProjectBuildingRequest();
-          configuration.setResolveDependencies(true);
-          return getMaven().readMavenProject(pomFile.getLocation().toFile(), configuration);
+      return execute(state, pomFiles.size() == 1 ? pomFiles.iterator().next() : null, resolverConfiguration,
+          (context, aMonitor) -> {
+        ProjectBuildingRequest configuration = context.newProjectBuildingRequest();
+        configuration.setResolveDependencies(true);
+        return getMaven().readMavenProjects(
+              pomFiles.stream().map(file -> file.getLocation().toFile()).collect(Collectors.toList()),
+              configuration);
       }, monitor);
     } catch(CoreException ex) {
-      DefaultMavenExecutionResult result = new DefaultMavenExecutionResult();
+      MavenExecutionResult result = new DefaultMavenExecutionResult();
       result.addException(ex);
-      return result;
+      return pomFiles.stream().filter(IResource::isAccessible).map(IResource::getLocation).filter(Objects::nonNull)
+          .map(IPath::toFile).collect(HashMap::new, (map, pomFile) -> map.put(pomFile, result),
+              (container, toFold) -> container.putAll(toFold));
     }
-
   }
 
   public IMavenProjectFacade[] getProjects() {
@@ -853,7 +890,9 @@ public class ProjectRegistryManager {
 
       /*package*/MavenExecutionRequest configureExecutionRequest(MavenExecutionRequest request, IProjectRegistry state,
           IFile pom, ResolverConfiguration resolverConfiguration) throws CoreException {
-    request.setPom(pom.getLocation().toFile());
+    if(pom != null) {
+      request.setPom(pom.getLocation().toFile());
+    }
 
     request.addActiveProfiles(resolverConfiguration.getActiveProfileList());
     request.addInactiveProfiles(resolverConfiguration.getInactiveProfileList());
@@ -1045,6 +1084,11 @@ public class ProjectRegistryManager {
     }
   }
 
+  /**
+   * Do not modify this map directly, use {@link #putMavenProject(MavenProjectFacade, MavenProject)}
+   *
+   * @return
+   */
   Map<MavenProjectFacade, MavenProject> getContextProjects() {
     Map<MavenProjectFacade, MavenProject> projects = null;
     MavenExecutionContext context = MavenExecutionContext.getThreadContext(false);