Bug 207802 [prov] Add multi-threaded download and mirror selection into p2's artifact retrieval

3 files changed, 124 insertions, 13 deletions

diff --git a/bundles/org.eclipse.equinox.p2.artifact.repository/META-INF/MANIFEST.MF b/bundles/org.eclipse.equinox.p2.artifact.repository/META-INF/MANIFEST.MF
index b12e2ea73..faeb77c5e 100644
--- a/bundles/org.eclipse.equinox.p2.artifact.repository/META-INF/MANIFEST.MF
+++ b/bundles/org.eclipse.equinox.p2.artifact.repository/META-INF/MANIFEST.MF
@@ -12,6 +12,7 @@ Export-Package: org.eclipse.equinox.internal.p2.artifact.mirror;x-internal:=true
  org.eclipse.equinox.p2.artifact.repository.processing,
  org.eclipse.equinox.spi.p2.artifact.repository
 Import-Package: javax.xml.parsers,
+ org.eclipse.core.runtime.jobs,
  org.eclipse.core.runtime.preferences;resolution:=optional,
  org.eclipse.equinox.app;version="1.0.0",
  org.eclipse.equinox.internal.p2.core.helpers,
diff --git a/bundles/org.eclipse.equinox.p2.artifact.repository/src/org/eclipse/equinox/internal/p2/artifact/repository/simple/DownloadJob.java b/bundles/org.eclipse.equinox.p2.artifact.repository/src/org/eclipse/equinox/internal/p2/artifact/repository/simple/DownloadJob.java
new file mode 100644
index 000000000..bd436ddc1
--- /dev/null
+++ b/bundles/org.eclipse.equinox.p2.artifact.repository/src/org/eclipse/equinox/internal/p2/artifact/repository/simple/DownloadJob.java
@@ -0,0 +1,82 @@
+/*******************************************************************************
+ * Copyright (c) 2008 Genuitec, LLC and others. All rights reserved. This
+ * program and the accompanying materials are made available under the terms of
+ * the Eclipse Public License v1.0 which accompanies this distribution, and is
+ * available at http://www.eclipse.org/legal/epl-v10.html
+ * 
+ * Contributors: Genuitec, LLC - initial API and implementation
+ * 						IBM Corporation - ongoing maintenance
+ ******************************************************************************/
+package org.eclipse.equinox.internal.p2.artifact.repository.simple;
+
+import java.util.LinkedList;
+import org.eclipse.core.runtime.*;
+import org.eclipse.core.runtime.jobs.Job;
+import org.eclipse.equinox.internal.p2.artifact.repository.ArtifactRequest;
+import org.eclipse.equinox.p2.artifact.repository.IArtifactRequest;
+import org.eclipse.osgi.util.NLS;
+
+public class DownloadJob extends Job {
+	static final Object FAMILY = new Object();
+
+	private LinkedList requestsPending;
+	private SimpleArtifactRepository repository;
+	private IProgressMonitor masterMonitor;
+	private MultiStatus overallStatus;
+
+	DownloadJob(String name) {
+		super(name);
+		setSystem(true);
+	}
+
+	void initialize(SimpleArtifactRepository repository, LinkedList requestsPending, IProgressMonitor masterMonitor, MultiStatus overallStatus) {
+		this.repository = repository;
+		this.requestsPending = requestsPending;
+		this.masterMonitor = masterMonitor;
+		this.overallStatus = overallStatus;
+	}
+
+	/* (non-Javadoc)
+	 * @see org.eclipse.core.runtime.jobs.Job#belongsTo(java.lang.Object)
+	 */
+	public boolean belongsTo(Object family) {
+		return family == FAMILY;
+	}
+
+	/* (non-Javadoc)
+	 * @see org.eclipse.core.runtime.jobs.Job#run(org.eclipse.core.runtime.IProgressMonitor)
+	 */
+	protected IStatus run(IProgressMonitor jobMonitor) {
+		jobMonitor.beginTask("Downloading artifacts", 1);
+		do {
+			// get the request we are going to process
+			IArtifactRequest request;
+			synchronized (requestsPending) {
+				if (requestsPending.isEmpty())
+					break;
+				request = (IArtifactRequest) requestsPending.removeFirst();
+			}
+			if (masterMonitor.isCanceled())
+				return Status.CANCEL_STATUS;
+
+			// prepare a progress monitor that reports to both the master monitor and for the job
+			IProgressMonitor monitor = new NullProgressMonitor();
+			// progress monitor updating from getArtifact() doesn't seem to be working
+			masterMonitor.subTask(NLS.bind("Downloading {0}.", request.getArtifactKey().getId()));
+
+			// process the actual request
+			IStatus status = repository.getArtifact((ArtifactRequest) request, monitor);
+			if (!status.isOK()) {
+				synchronized (overallStatus) {
+					overallStatus.add(status);
+				}
+			}
+
+			// update progress
+			masterMonitor.worked(1);
+		} while (true);
+
+		jobMonitor.done();
+		return Status.OK_STATUS;
+	}
+}
diff --git a/bundles/org.eclipse.equinox.p2.artifact.repository/src/org/eclipse/equinox/internal/p2/artifact/repository/simple/SimpleArtifactRepository.java b/bundles/org.eclipse.equinox.p2.artifact.repository/src/org/eclipse/equinox/internal/p2/artifact/repository/simple/SimpleArtifactRepository.java
index 29e68da8e..38c20ebc2 100644
--- a/bundles/org.eclipse.equinox.p2.artifact.repository/src/org/eclipse/equinox/internal/p2/artifact/repository/simple/SimpleArtifactRepository.java
+++ b/bundles/org.eclipse.equinox.p2.artifact.repository/src/org/eclipse/equinox/internal/p2/artifact/repository/simple/SimpleArtifactRepository.java
@@ -5,6 +5,7 @@
  * available at http://www.eclipse.org/legal/epl-v10.html
  * 
  * Contributors: IBM Corporation - initial API and implementation
+ * 						Genuitec, LLC - support for multi-threaded downloads
  ******************************************************************************/
 package org.eclipse.equinox.internal.p2.artifact.repository.simple;
 
@@ -15,6 +16,7 @@ import java.util.*;
 import java.util.jar.JarEntry;
 import java.util.jar.JarOutputStream;
 import org.eclipse.core.runtime.*;
+import org.eclipse.core.runtime.jobs.Job;
 import org.eclipse.equinox.internal.p2.artifact.repository.*;
 import org.eclipse.equinox.internal.p2.core.helpers.FileUtils;
 import org.eclipse.equinox.internal.p2.core.helpers.ServiceHelper;
@@ -24,6 +26,7 @@ import org.eclipse.equinox.p2.artifact.repository.processing.ProcessingStepHandl
 import org.eclipse.equinox.p2.core.ProvisionException;
 import org.eclipse.equinox.p2.metadata.IArtifactKey;
 import org.eclipse.equinox.spi.p2.artifact.repository.AbstractArtifactRepository;
+import org.eclipse.osgi.util.NLS;
 
 public class SimpleArtifactRepository extends AbstractArtifactRepository implements IArtifactRepository, IFileArtifactRepository {
 
@@ -379,7 +382,7 @@ public class SimpleArtifactRepository extends AbstractArtifactRepository impleme
 		return super.getAdapter(adapter);
 	}
 
-	private IStatus getArtifact(ArtifactRequest request, IProgressMonitor monitor) {
+	IStatus getArtifact(ArtifactRequest request, IProgressMonitor monitor) {
 		request.setSourceRepository(this);
 		request.perform(monitor);
 		return request.getResult();
@@ -429,20 +432,45 @@ public class SimpleArtifactRepository extends AbstractArtifactRepository impleme
 	}
 
 	public IStatus getArtifacts(IArtifactRequest[] requests, IProgressMonitor monitor) {
-		SubMonitor subMonitor = SubMonitor.convert(monitor, requests.length);
-		try {
-			MultiStatus overallStatus = new MultiStatus(Activator.ID, IStatus.OK, null, null);
-			for (int i = 0; i < requests.length; i++) {
-				if (monitor.isCanceled())
-					return Status.CANCEL_STATUS;
-				IStatus result = getArtifact((ArtifactRequest) requests[i], subMonitor.newChild(1));
-				if (!result.isOK())
-					overallStatus.add(result);
+		final MultiStatus overallStatus = new MultiStatus(Activator.ID, IStatus.OK, null, null);
+		LinkedList requestsPending = new LinkedList(Arrays.asList(requests));
+
+		// TODO : Determine number of threads to use from a property
+		int numberOfJobs = Math.min(requests.length, 4); // magic number
+		if (numberOfJobs <= 1) {
+			SubMonitor subMonitor = SubMonitor.convert(monitor, requests.length);
+			try {
+				for (int i = 0; i < requests.length; i++) {
+					if (monitor.isCanceled())
+						return Status.CANCEL_STATUS;
+					IStatus result = getArtifact((ArtifactRequest) requests[i], subMonitor.newChild(1));
+					if (!result.isOK())
+						overallStatus.add(result);
+				}
+			} finally {
+				subMonitor.done();
+			}
+		} else {
+			// initialize the various jobs needed to process the get artifact requests
+			monitor.beginTask(NLS.bind("Download {0} artifacts", Integer.toString(requests.length)), requests.length);
+			try {
+				DownloadJob jobs[] = new DownloadJob[numberOfJobs];
+				for (int i = 0; i < numberOfJobs; i++) {
+					jobs[i] = new DownloadJob("Install download " + i);
+					jobs[i].initialize(this, requestsPending, monitor, overallStatus);
+					jobs[i].schedule();
+				}
+				// wait for all the jobs to complete
+				try {
+					Job.getJobManager().join(DownloadJob.FAMILY, null);
+				} catch (InterruptedException e) {
+					//ignore
+				}
+			} finally {
+				monitor.done();
 			}
-			return (monitor.isCanceled() ? Status.CANCEL_STATUS : overallStatus);
-		} finally {
-			subMonitor.done();
 		}
+		return (monitor.isCanceled() ? Status.CANCEL_STATUS : overallStatus);
 	}
 
 	public synchronized IArtifactDescriptor getCompleteArtifactDescriptor(IArtifactKey key) {