Bug 540507 - parallel activation of bundles from framework start-level

By default the framework start-level implementation activates all
bundles that have the same start-level in sequential order using the
bundle ID (install order) to determine the order the bundles are started
from a single thread.

A new configuration option is added equinox.start.level.thread.count
that is set to 1 by default.  This will cause the framework to continue
with the behavior of activating all bundles with a single thread.

If equinox.start.level.thread.count is set to -1 then the framework will
use an executor with the number of threads == to
java.lang.Runtime.availableProcessors() to activate the bundles in
parallel.  This means that the order the bundles with the same
start-level are activated becomes random.  The
equinox.start.level.thread.count config setting can also be set to a
value > 1 to give an explicit value for the number of threads to use
(instead of using java.lang.Runtime.availableProcessors() to determine
the number)

Change-Id: Ifcf6f18b762bcb114f8f8c75630fdca0cb79a24c
Signed-off-by: Thomas Watson <tjwatson@us.ibm.com>

7 files changed, 148 insertions, 47 deletions

diff --git a/bundles/org.eclipse.osgi/META-INF/MANIFEST.MF b/bundles/org.eclipse.osgi/META-INF/MANIFEST.MF
index dc7a293e5..1f3e297a4 100644
--- a/bundles/org.eclipse.osgi/META-INF/MANIFEST.MF
+++ b/bundles/org.eclipse.osgi/META-INF/MANIFEST.MF
@@ -3,7 +3,7 @@ Bundle-ManifestVersion: 2
 Export-Package: org.eclipse.core.runtime.adaptor;x-friends:="org.eclipse.core.runtime",
  org.eclipse.core.runtime.internal.adaptor;x-internal:=true,
  org.eclipse.equinox.log;version="1.1";uses:="org.osgi.framework,org.osgi.service.log",
- org.eclipse.osgi.container;version="1.3";
+ org.eclipse.osgi.container;version="1.4";
   uses:="org.eclipse.osgi.report.resolution,
    org.osgi.framework.wiring,
    org.osgi.framework.startlevel,
@@ -101,7 +101,7 @@ Bundle-Activator: org.eclipse.osgi.internal.framework.SystemBundleActivator
 Bundle-Description: %systemBundle
 Bundle-Copyright: %copyright
 Bundle-Vendor: %eclipse.org
-Bundle-Version: 3.13.400.qualifier
+Bundle-Version: 3.14.0.qualifier
 Bundle-Localization: systembundle
 Bundle-DocUrl: http://www.eclipse.org
 Eclipse-ExtensibleAPI: true
diff --git a/bundles/org.eclipse.osgi/container/src/org/eclipse/osgi/container/ModuleContainer.java b/bundles/org.eclipse.osgi/container/src/org/eclipse/osgi/container/ModuleContainer.java
index e7870771f..e51ef84e8 100644
--- a/bundles/org.eclipse.osgi/container/src/org/eclipse/osgi/container/ModuleContainer.java
+++ b/bundles/org.eclipse.osgi/container/src/org/eclipse/osgi/container/ModuleContainer.java
@@ -26,6 +26,8 @@ import java.util.List;
 import java.util.ListIterator;
 import java.util.Map;
 import java.util.Set;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.Executor;
 import java.util.concurrent.Semaphore;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
@@ -1563,7 +1565,7 @@ public final class ModuleContainer implements DebugOptionsListener {
 		private final Object eventManagerLock = new Object();
 		private EventManager startLevelThread = null;
 		private final Object frameworkStartLevelLock = new Object();
-		private boolean debugStartLevel = false;
+		boolean debugStartLevel = false;
 		{
 			setDebugOptions();
 		}
@@ -1708,6 +1710,8 @@ public final class ModuleContainer implements DebugOptionsListener {
 					List<Module> sorted = null;
 					long currentTimestamp = Long.MIN_VALUE;
 					if (newStartLevel > currentSL) {
+						List<Module> lazyStart = null;
+						List<Module> eagerStart = null;
 						for (int i = currentSL; i < newStartLevel; i++) {
 							int toStartLevel = i + 1;
 							activeStartLevel.set(toStartLevel);
@@ -1718,12 +1722,15 @@ public final class ModuleContainer implements DebugOptionsListener {
 								moduleDatabase.readLock();
 								try {
 									sorted = moduleDatabase.getSortedModules(Sort.BY_START_LEVEL);
+									lazyStart = new ArrayList<>(sorted.size());
+									eagerStart = new ArrayList<>(sorted.size());
+									separateModulesByActivationPolicy(sorted, lazyStart, eagerStart);
 									currentTimestamp = moduleDatabase.getTimestamp();
 								} finally {
 									moduleDatabase.readUnlock();
 								}
 							}
-							incStartLevel(toStartLevel, sorted);
+							incStartLevel(toStartLevel, lazyStart, eagerStart);
 						}
 					} else {
 						for (int i = currentSL; i > newStartLevel; i--) {
@@ -1759,13 +1766,27 @@ public final class ModuleContainer implements DebugOptionsListener {
 			}
 		}
 
-		private void incStartLevel(int toStartLevel, List<Module> sortedModules) {
-			incStartLevel(toStartLevel, sortedModules, true);
-			incStartLevel(toStartLevel, sortedModules, false);
+		private void incStartLevel(int toStartLevel, List<Module> lazyStart, List<Module> eagerStart) {
+			incStartLevel(toStartLevel, lazyStart);
+			incStartLevel(toStartLevel, eagerStart);
 		}
 
-		private void incStartLevel(int toStartLevel, List<Module> sortedModules, boolean lazyOnly) {
+		private void separateModulesByActivationPolicy(List<Module> sortedModules, List<Module> lazyStart, List<Module> eagerStart) {
 			for (Module module : sortedModules) {
+				if (module.isLazyActivate()) {
+					lazyStart.add(module);
+				} else {
+					eagerStart.add(module);
+				}
+			}
+		}
+
+		private void incStartLevel(final int toStartLevel, List<Module> candidatesToStart) {
+			if (candidatesToStart.isEmpty()) {
+				return;
+			}
+			List<Module> toStart = new ArrayList<>();
+			for (final Module module : candidatesToStart) {
 				if (isRefreshingSystemModule()) {
 					return;
 				}
@@ -1775,22 +1796,8 @@ public final class ModuleContainer implements DebugOptionsListener {
 						// skip modules who should have already been started
 						continue;
 					} else if (moduleStartLevel == toStartLevel) {
-						boolean isLazyStart = module.isLazyActivate();
-						if (lazyOnly ? isLazyStart : !isLazyStart) {
-							if (debugStartLevel) {
-								Debug.println("StartLevel: resuming bundle; " + toString(module) + "; with startLevel=" + moduleStartLevel); //$NON-NLS-1$ //$NON-NLS-2$
-							}
-							try {
-								module.start(StartOptions.TRANSIENT_IF_AUTO_START, StartOptions.TRANSIENT_RESUME);
-							} catch (BundleException e) {
-								adaptor.publishContainerEvent(ContainerEvent.ERROR, module, e);
-							} catch (IllegalStateException e) {
-								// been uninstalled
-								continue;
-							}
-						}
+						toStart.add(module);
 					} else {
-						// can stop resuming since any remaining modules have a greater startlevel than the active startlevel
 						break;
 					}
 				} catch (IllegalStateException e) {
@@ -1798,6 +1805,36 @@ public final class ModuleContainer implements DebugOptionsListener {
 					continue;
 				}
 			}
+			if (toStart.isEmpty()) {
+				return;
+			}
+			final Executor executor = adaptor.getStartLevelExecutor();
+			final CountDownLatch done = new CountDownLatch(toStart.size());
+			for (final Module module : toStart) {
+				executor.execute(new Runnable() {
+					@Override
+					public void run() {
+						try {
+							if (debugStartLevel) {
+								Debug.println("StartLevel: resuming bundle; " + ContainerStartLevel.this.toString(module) + "; with startLevel=" + toStartLevel); //$NON-NLS-1$ //$NON-NLS-2$
+							}
+							module.start(StartOptions.TRANSIENT_IF_AUTO_START, StartOptions.TRANSIENT_RESUME);
+						} catch (BundleException e) {
+							adaptor.publishContainerEvent(ContainerEvent.ERROR, module, e);
+						} catch (IllegalStateException e) {
+							// been uninstalled
+						} finally {
+							done.countDown();
+						}
+					}
+				});
+
+			}
+			try {
+				done.await();
+			} catch (InterruptedException e) {
+				adaptor.publishContainerEvent(ContainerEvent.ERROR, moduleDatabase.getModule(0), e);
+			}
 		}
 
 		private void decStartLevel(int toStartLevel, List<Module> sortedModules) {
@@ -1865,7 +1902,7 @@ public final class ModuleContainer implements DebugOptionsListener {
 			}
 		}
 
-		private String toString(Module m) {
+		String toString(Module m) {
 			Bundle b = m.getBundle();
 			return b != null ? b.toString() : m.toString();
 		}
diff --git a/bundles/org.eclipse.osgi/container/src/org/eclipse/osgi/container/ModuleContainerAdaptor.java b/bundles/org.eclipse.osgi/container/src/org/eclipse/osgi/container/ModuleContainerAdaptor.java
index bfa760e79..df8838e76 100644
--- a/bundles/org.eclipse.osgi/container/src/org/eclipse/osgi/container/ModuleContainerAdaptor.java
+++ b/bundles/org.eclipse.osgi/container/src/org/eclipse/osgi/container/ModuleContainerAdaptor.java
@@ -22,12 +22,20 @@ import org.eclipse.osgi.service.debug.DebugOptions;
 import org.osgi.framework.Constants;
 import org.osgi.framework.FrameworkListener;
 import org.osgi.framework.hooks.resolver.ResolverHookFactory;
+import org.osgi.framework.startlevel.FrameworkStartLevel;
 
 /**
  * Adapts the behavior of a container.
  * @since 3.10
  */
 public abstract class ModuleContainerAdaptor {
+	private static Executor defaultExecutor = new Executor() {
+		@Override
+		public void execute(Runnable command) {
+			command.run();
+		}
+	};
+
 	/**
 	 * Event types that may be {@link #publishContainerEvent(ContainerEvent, Module, Throwable, FrameworkListener...) published}
 	 * for a container.
@@ -291,12 +299,19 @@ public abstract class ModuleContainerAdaptor {
 	 * @since 3.11
 	 */
 	public Executor getResolverExecutor() {
-		return new Executor() {
-			@Override
-			public void execute(Runnable command) {
-				command.run();
-			}
-		};
+		return defaultExecutor;
+	}
+
+	/**
+	 * Returns the executor used to by the 
+	 * {@link ModuleContainer#getFrameworkStartLevel() FrameworkStartLevel} implementation to
+	 * start bundles that have the same start level.  This allows bundles to be
+	 * started in parallel.
+	 * @return the executor used by the {@link FrameworkStartLevel} implementation.
+	 * @since 3.14
+	 */
+	public Executor getStartLevelExecutor() {
+		return defaultExecutor;
 	}
 
 	/**
diff --git a/bundles/org.eclipse.osgi/container/src/org/eclipse/osgi/internal/framework/EquinoxConfiguration.java b/bundles/org.eclipse.osgi/container/src/org/eclipse/osgi/internal/framework/EquinoxConfiguration.java
index dd8b559d6..f1f5b43e7 100644
--- a/bundles/org.eclipse.osgi/container/src/org/eclipse/osgi/internal/framework/EquinoxConfiguration.java
+++ b/bundles/org.eclipse.osgi/container/src/org/eclipse/osgi/internal/framework/EquinoxConfiguration.java
@@ -223,7 +223,10 @@ public class EquinoxConfiguration implements EnvironmentInfo {
 	public static final String PROP_ALLOW_RESTRICTED_PROVIDES = "osgi.equinox.allow.restricted.provides"; //$NON-NLS-1$
 	public static final String PROP_LOG_HISTORY_MAX = "equinox.log.history.max"; //$NON-NLS-1$
 
+	@Deprecated
 	public static final String PROP_RESOLVER_THREAD_COUNT = "equinox.resolver.thead.count"; //$NON-NLS-1$
+	public static final String PROP_EQUINOX_RESOLVER_THREAD_COUNT = "equinox.resolver.thread.count"; //$NON-NLS-1$
+	public static final String PROP_EQUINOX_START_LEVEL_THREAD_COUNT = "equinox.start.level.thread.count"; //$NON-NLS-1$
 	public static final String PROP_RESOLVER_REVISION_BATCH_SIZE = "equinox.resolver.revision.batch.size"; //$NON-NLS-1$
 	public static final String PROP_RESOLVER_BATCH_TIMEOUT = "equinox.resolver.batch.timeout"; //$NON-NLS-1$
 
diff --git a/bundles/org.eclipse.osgi/container/src/org/eclipse/osgi/internal/framework/EquinoxContainerAdaptor.java b/bundles/org.eclipse.osgi/container/src/org/eclipse/osgi/internal/framework/EquinoxContainerAdaptor.java
index 48b60316f..d8b7b2584 100644
--- a/bundles/org.eclipse.osgi/container/src/org/eclipse/osgi/internal/framework/EquinoxContainerAdaptor.java
+++ b/bundles/org.eclipse.osgi/container/src/org/eclipse/osgi/internal/framework/EquinoxContainerAdaptor.java
@@ -64,7 +64,8 @@ public class EquinoxContainerAdaptor extends ModuleContainerAdaptor {
 			platformClassLoader = (ClassLoader) getPlatformClassLoader.invoke(null);
 		} catch (Throwable t) {
 			// try everything possible to not fail <clinit>
-			platformClassLoader = new ClassLoader(Object.class.getClassLoader()) { /* boot class loader */};
+			platformClassLoader = new ClassLoader(Object.class.getClassLoader()) {
+				/* boot class loader */};
 		}
 		BOOT_CLASSLOADER = platformClassLoader;
 	}
@@ -76,8 +77,10 @@ public class EquinoxContainerAdaptor extends ModuleContainerAdaptor {
 	private final ClassLoader moduleClassLoaderParent;
 	private final AtomicLong lastSecurityAdminFlush;
 
-	final AtomicLazyInitializer<Executor> executor = new AtomicLazyInitializer<>();
-	final Callable<Executor> lazyExecutorCreator;
+	final AtomicLazyInitializer<Executor> resolverExecutor;
+	final Callable<Executor> lazyResolverExecutorCreator;
+	final AtomicLazyInitializer<Executor> startLevelExecutor;
+	final Callable<Executor> lazyStartLevelExecutorCreator;
 
 	public EquinoxContainerAdaptor(EquinoxContainer container, Storage storage, Map<Long, Generation> initial) {
 		this.container = container;
@@ -86,19 +89,53 @@ public class EquinoxContainerAdaptor extends ModuleContainerAdaptor {
 		this.initial = initial;
 		this.moduleClassLoaderParent = getModuleClassLoaderParent(container.getConfiguration());
 		this.lastSecurityAdminFlush = new AtomicLong();
-		this.lazyExecutorCreator = createLazyExecutorCreator(container.getConfiguration());
-	}
 
-	private Callable<Executor> createLazyExecutorCreator(EquinoxConfiguration config) {
-		String threadCntProp = config.getConfiguration(EquinoxConfiguration.PROP_RESOLVER_THREAD_COUNT);
-		int threadCntTmp;
+		EquinoxConfiguration config = container.getConfiguration();
+		@SuppressWarnings("deprecation")
+		String resolverThreadCntProp = config.getConfiguration(EquinoxConfiguration.PROP_EQUINOX_RESOLVER_THREAD_COUNT, //
+				config.getConfiguration(EquinoxConfiguration.PROP_RESOLVER_THREAD_COUNT));
+
+		int resolverThreadCnt;
+		try {
+			// note that resolver thread count defaults to -1 (compute based on processor number)
+			resolverThreadCnt = resolverThreadCntProp == null ? -1 : Integer.parseInt(resolverThreadCntProp);
+		} catch (NumberFormatException e) {
+			resolverThreadCnt = -1;
+		}
+		String startLevelThreadCntProp = config.getConfiguration(EquinoxConfiguration.PROP_EQUINOX_START_LEVEL_THREAD_COUNT);
+		int startLevelThreadCnt;
 		try {
-			threadCntTmp = threadCntProp == null ? -1 : Integer.parseInt(threadCntProp);
+			// Note that start-level thread count defaults to 1 (synchronous start)
+			startLevelThreadCnt = startLevelThreadCntProp == null ? 1 : Integer.parseInt(startLevelThreadCntProp);
 		} catch (NumberFormatException e) {
-			threadCntTmp = -1;
+			startLevelThreadCnt = 1;
+		}
+
+		if (resolverThreadCnt == startLevelThreadCnt) {
+			// use a single executor
+			this.resolverExecutor = new AtomicLazyInitializer<>();
+			this.lazyResolverExecutorCreator = createLazyExecutorCreator(container.getConfiguration(), //
+					"Equinox executor thread - " + EquinoxContainerAdaptor.this.toString(), //$NON-NLS-1$
+					resolverThreadCnt);
+			this.startLevelExecutor = resolverExecutor;
+			this.lazyStartLevelExecutorCreator = lazyResolverExecutorCreator;
+		} else {
+			// use two different executors
+			this.resolverExecutor = new AtomicLazyInitializer<>();
+			this.lazyResolverExecutorCreator = createLazyExecutorCreator(container.getConfiguration(), //
+					"Equinox resolver thread - " + EquinoxContainerAdaptor.this.toString(), //$NON-NLS-1$
+					resolverThreadCnt);
+			this.startLevelExecutor = new AtomicLazyInitializer<>();
+			this.lazyStartLevelExecutorCreator = createLazyExecutorCreator(container.getConfiguration(), //
+					"Equinox start level thread - " + EquinoxContainerAdaptor.this.toString(), //$NON-NLS-1$
+					startLevelThreadCnt);
 		}
+
+	}
+
+	private Callable<Executor> createLazyExecutorCreator(EquinoxConfiguration config, final String threadName, int threadCnt) {
 		// use the number of processors - 1 because we use the current thread when rejected
-		final int maxThreads = threadCntTmp <= 0 ? Math.max(Runtime.getRuntime().availableProcessors() - 1, 1) : threadCntTmp;
+		final int maxThreads = threadCnt <= 0 ? Math.max(Runtime.getRuntime().availableProcessors() - 1, 1) : threadCnt;
 		return new Callable<Executor>() {
 			@Override
 			public Executor call() throws Exception {
@@ -120,7 +157,7 @@ public class EquinoxContainerAdaptor extends ModuleContainerAdaptor {
 				ThreadFactory threadFactory = new ThreadFactory() {
 					@Override
 					public Thread newThread(Runnable r) {
-						Thread t = new Thread(r, "Resolver thread - " + EquinoxContainerAdaptor.this.toString()); //$NON-NLS-1$
+						Thread t = new Thread(r, threadName);
 						t.setDaemon(true);
 						return t;
 					}
@@ -362,7 +399,12 @@ public class EquinoxContainerAdaptor extends ModuleContainerAdaptor {
 
 	@Override
 	public Executor getResolverExecutor() {
-		return executor.getInitialized(lazyExecutorCreator);
+		return resolverExecutor.getInitialized(lazyResolverExecutorCreator);
+	}
+
+	@Override
+	public Executor getStartLevelExecutor() {
+		return startLevelExecutor.getInitialized(lazyStartLevelExecutorCreator);
 	}
 
 	@Override
@@ -370,8 +412,12 @@ public class EquinoxContainerAdaptor extends ModuleContainerAdaptor {
 		return container.getScheduledExecutor();
 	}
 
-	public void shutdownResolverExecutor() {
-		Executor current = executor.getAndClear();
+	public void shutdownExecutors() {
+		Executor current = resolverExecutor.getAndClear();
+		if (current instanceof ExecutorService) {
+			((ExecutorService) current).shutdown();
+		}
+		current = startLevelExecutor.getAndClear();
 		if (current instanceof ExecutorService) {
 			((ExecutorService) current).shutdown();
 		}
diff --git a/bundles/org.eclipse.osgi/container/src/org/eclipse/osgi/storage/Storage.java b/bundles/org.eclipse.osgi/container/src/org/eclipse/osgi/storage/Storage.java
index 25c70ac8f..ddea8b6f5 100644
--- a/bundles/org.eclipse.osgi/container/src/org/eclipse/osgi/storage/Storage.java
+++ b/bundles/org.eclipse.osgi/container/src/org/eclipse/osgi/storage/Storage.java
@@ -474,7 +474,7 @@ public class Storage {
 			}
 		}
 		mruList.shutdown();
-		adaptor.shutdownResolverExecutor();
+		adaptor.shutdownExecutors();
 	}
 
 	private boolean needUpdate(ModuleRevision currentRevision, ModuleRevisionBuilder newBuilder) {
diff --git a/bundles/org.eclipse.osgi/pom.xml b/bundles/org.eclipse.osgi/pom.xml
index 92a400413..48467d543 100644
--- a/bundles/org.eclipse.osgi/pom.xml
+++ b/bundles/org.eclipse.osgi/pom.xml
@@ -19,7 +19,7 @@
   </parent>
   <groupId>org.eclipse.osgi</groupId>
   <artifactId>org.eclipse.osgi</artifactId>
-  <version>3.13.400-SNAPSHOT</version>
+  <version>3.14.0-SNAPSHOT</version>
   <packaging>eclipse-plugin</packaging>
 
   <build>