[Releng] Change P2Indexer to use SAXParsers

2 files changed, 99 insertions, 62 deletions

diff --git a/plugins/org.eclipse.oomph.p2.core/P2 Indexer.launch b/plugins/org.eclipse.oomph.p2.core/P2 Indexer.launch
index 211ccda68..782d91740 100644
--- a/plugins/org.eclipse.oomph.p2.core/P2 Indexer.launch
+++ b/plugins/org.eclipse.oomph.p2.core/P2 Indexer.launch
@@ -27,7 +27,7 @@
 </listAttribute>
 <booleanAttribute key="org.eclipse.jdt.launching.ATTR_USE_START_ON_FIRST_THREAD" value="true"/>
 <stringAttribute key="org.eclipse.jdt.launching.JRE_CONTAINER" value="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.8"/>
-<stringAttribute key="org.eclipse.jdt.launching.PROGRAM_ARGUMENTS" value="-consoleLog&#13;&#10;-noSplash&#13;&#10;C:\Users\Stepper\Desktop\indexing&#13;&#10;24&#13;&#10;http://download.eclipse.org&#13;&#10;${system_property:java.io.tmpdir}index&#13;&#10;-verboseX&#13;&#10;-report C:\Users\Stepper\AppData\Local\Temp\index\_report_"/>
+<stringAttribute key="org.eclipse.jdt.launching.PROGRAM_ARGUMENTS" value="-consoleLog&#13;&#10;-noSplash&#13;&#10;C:\Users\Stepper\Desktop\download.eclipse.org&#13;&#10;24&#13;&#10;http://download.eclipse.org&#13;&#10;${system_property:java.io.tmpdir}index&#13;&#10;-verboseX&#13;&#10;-report C:\Users\Stepper\AppData\Local\Temp\index\_report_"/>
 <stringAttribute key="org.eclipse.jdt.launching.SOURCE_PATH_PROVIDER" value="org.eclipse.pde.ui.workbenchClasspathProvider"/>
 <stringAttribute key="org.eclipse.jdt.launching.VM_ARGUMENTS" value="-Xmx6g&#13;&#10;-Dfile.encoding=UTF-8&#13;&#10;-Dorg.eclipse.emf.ecore.plugin.EcorePlugin.doNotLoadResourcesPlugin=true&#13;&#10;-Dsun.zip.disableMemoryMapping=true&#13;&#10;-Dprojects.txt=C:\Users\Stepper\Desktop\projects.txt&#13;&#10;-Dmappings.txt=C:\Users\Stepper\Desktop\mappings.txt"/>
 <stringAttribute key="pde.version" value="3.3"/>
diff --git a/plugins/org.eclipse.oomph.p2.core/src/org/eclipse/oomph/p2/internal/core/P2Indexer.java b/plugins/org.eclipse.oomph.p2.core/src/org/eclipse/oomph/p2/internal/core/P2Indexer.java
index 6caa3a78e..f0ce5a9fb 100644
--- a/plugins/org.eclipse.oomph.p2.core/src/org/eclipse/oomph/p2/internal/core/P2Indexer.java
+++ b/plugins/org.eclipse.oomph.p2.core/src/org/eclipse/oomph/p2/internal/core/P2Indexer.java
@@ -64,6 +64,7 @@ import java.util.StringTokenizer;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentLinkedDeque;
 import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.Future;
@@ -159,15 +160,20 @@ public final class P2Indexer implements IApplication
         future.get();
       }
 
+      System.out.println();
+      System.out.println("Generating index to " + outputFolder);
       generateIndex(outputFolder);
 
       if (reporter != null)
       {
+        System.out.println();
+        System.out.println("Writing report to " + reporter.getReportFolder());
         reporter.writeReport(this);
       }
     }
     finally
     {
+      System.out.println();
       System.out.println("Took " + (System.currentTimeMillis() - start) / 1000 + " seconds.");
       threadPool.shutdown();
     }
@@ -323,9 +329,8 @@ public final class P2Indexer implements IApplication
 
   private void generateRepositoryMetadata()
   {
-    for (final Map.Entry<URI, Repository> entry : repositories.entrySet())
+    for (final Repository repository : repositories.values())
     {
-      final Repository repository = entry.getValue();
       scheduleTask(new Runnable()
       {
         public void run()
@@ -344,8 +349,13 @@ public final class P2Indexer implements IApplication
           }
           catch (Exception ex)
           {
-            repositories.remove(entry.getKey());
-            error("Processing " + repository.getMetadataFile(), ex);
+            String message = getStackTrace(ex);
+            if (reporter != null)
+            {
+              reporter.reportError(repository, message);
+            }
+
+            System.err.println(repository.getURI() + " --> " + message);
           }
           finally
           {
@@ -359,22 +369,6 @@ public final class P2Indexer implements IApplication
     }
   }
 
-  private void error(String message, Exception exception)
-  {
-    try
-    {
-      ByteArrayOutputStream bytes = new ByteArrayOutputStream();
-      PrintStream out = new PrintStream(bytes);
-      out.println(message);
-      exception.printStackTrace(out);
-      System.err.write(bytes.toByteArray());
-    }
-    catch (Exception ex1)
-    {
-      //$FALL-THROUGH$
-    }
-  }
-
   private void scheduleTask(Runnable task)
   {
     deque.addLast(threadPool.submit(task));
@@ -495,66 +489,86 @@ public final class P2Indexer implements IApplication
     }
   }
 
-  private int writeCapabilities(File outputFolder)
+  private int writeCapabilities(final File outputFolder) throws InterruptedException
   {
-    int count = 0;
-    for (Map.Entry<String, List<Capability>> entry : capabilities.entrySet())
+    int count = capabilities.size();
+    final CountDownLatch finished = new CountDownLatch(count);
+
+    for (final Map.Entry<String, List<Capability>> entry : capabilities.entrySet())
     {
       ++count;
-      String name = entry.getKey();
-      if (verbose)
+      threadPool.submit(new Runnable()
       {
-        System.out.println("Capability " + name);
-      }
-
-      Map<Repository, Set<String>> versions = new HashMap<Repository, Set<String>>();
-      for (Capability capability : entry.getValue())
-      {
-        Repository repository = capability.getRepository();
-        if (repositories.containsKey(repository.getURI()))
+        public void run()
         {
-          Set<String> set = versions.get(repository);
-          if (set == null)
+          try
           {
-            set = new HashSet<String>();
-            versions.put(repository, set);
+            writeCapability(outputFolder, entry.getKey(), entry.getValue());
+          }
+          finally
+          {
+            finished.countDown();
           }
-
-          set.add(capability.getVersion());
         }
-      }
+      });
+    }
 
-      List<String> lines = new ArrayList<String>();
-      lines.add(Long.toString(timeStamp));
+    finished.await();
+    return count;
+  }
 
-      for (Map.Entry<Repository, Set<String>> versionEntry : versions.entrySet())
-      {
-        Repository repository = versionEntry.getKey();
-        StringBuilder builder = new StringBuilder();
-        builder.append(repository.getID());
+  private void writeCapability(File outputFolder, String name, List<Capability> capabilities)
+  {
+    if (verbose)
+    {
+      System.out.println("Capability " + name);
+    }
 
-        for (String version : versionEntry.getValue())
+    Map<Repository, Set<String>> versions = new HashMap<Repository, Set<String>>();
+    for (Capability capability : capabilities)
+    {
+      Repository repository = capability.getRepository();
+      if (repositories.containsKey(repository.getURI()))
+      {
+        Set<String> set = versions.get(repository);
+        if (set == null)
         {
-          builder.append(",");
-          builder.append(version);
+          set = new HashSet<String>();
+          versions.put(repository, set);
         }
 
-        lines.add(builder.toString());
+        set.add(capability.getVersion());
       }
+    }
 
-      try
-      {
-        File file = new File(outputFolder, name);
-        file.getParentFile().mkdirs();
-        IOUtil.writeLines(file, CHARSET, lines);
-      }
-      catch (Exception ex)
+    List<String> lines = new ArrayList<String>();
+    lines.add(Long.toString(timeStamp));
+
+    for (Map.Entry<Repository, Set<String>> versionEntry : versions.entrySet())
+    {
+      Repository repository = versionEntry.getKey();
+      StringBuilder builder = new StringBuilder();
+      builder.append(repository.getID());
+
+      for (String version : versionEntry.getValue())
       {
-        error("Capability " + name, ex);
+        builder.append(",");
+        builder.append(version);
       }
+
+      lines.add(builder.toString());
     }
 
-    return count;
+    try
+    {
+      File file = new File(outputFolder, name);
+      file.getParentFile().mkdirs();
+      IOUtil.writeLines(file, CHARSET, lines);
+    }
+    catch (Exception ex)
+    {
+      System.err.println("Capability " + name + " --> " + getStackTrace(ex));
+    }
   }
 
   /**
@@ -572,6 +586,21 @@ public final class P2Indexer implements IApplication
     }
   }
 
+  private static String getStackTrace(Exception exception)
+  {
+    try
+    {
+      ByteArrayOutputStream bytes = new ByteArrayOutputStream();
+      PrintStream out = new PrintStream(bytes);
+      exception.printStackTrace(out);
+      return bytes.toString();
+    }
+    catch (Exception ex)
+    {
+      return ex.getMessage();
+    }
+  }
+
   /**
    * @author Eike Stepper
    */
@@ -1021,6 +1050,11 @@ public final class P2Indexer implements IApplication
       projectMapper = new ProjectMapper(baseURI.toString());
     }
 
+    public File getReportFolder()
+    {
+      return reportFolder;
+    }
+
     public synchronized void reportRepository(Repository repository)
     {
       String projectID = projectMapper.getProjectID(repository.getURI());
@@ -1331,7 +1365,10 @@ public final class P2Indexer implements IApplication
             Repository repository = entry.getKey();
             List<String> errors = entry.getValue();
 
-            int id = ids.get(repository).getElement2();
+            Pair<Project, Integer> pair = ids.get(repository);
+            Integer idWrapper = pair.getElement2();
+            int id = idWrapper;
+
             writer.write("<h2><a name=\"repo" + id + "\">" + (errors.isEmpty() ? "" : "<a name=\"err" + erroneousRepo + "\">")
                 + (repository.isComposed() ? "Composite" : "Simple") + " Repository <a href=\"" + repository.getURI() + "\">" + repository.getURI()
                 + "</a></h2>\n");