From a2f3feaa800d097b5f41348d3ff3458d07d75fdc Mon Sep 17 00:00:00 2001
From: Andrey Loskutov
Date: Mon, 3 Sep 2018 21:00:12 +0200
Subject: Bug 355011 - Deadlock on creating console in the build thread

Do not create console directly in the launchChanged() callback, because
the current thread may hold already some lock, and if the
ProcessConsole() init requires another one lock, we will have a classic
deadlock. In case of bug 355011 current thread owns workspace lock and
ProcessConsole() init requires UI lock.

Change-Id: Ia01df4cb3f714b16fce84620538e66aa3de10b76
Signed-off-by: Andrey Loskutov <loskutov@gmx.de>
---
 .../ui/views/console/ProcessConsoleManager.java    | 77 +++++++++++++++++-----
 1 file changed, 61 insertions(+), 16 deletions(-)

(limited to 'org.eclipse.debug.ui')

diff --git a/org.eclipse.debug.ui/ui/org/eclipse/debug/internal/ui/views/console/ProcessConsoleManager.java b/org.eclipse.debug.ui/ui/org/eclipse/debug/internal/ui/views/console/ProcessConsoleManager.java
index e8241c054..fb73e6e04 100644
--- a/org.eclipse.debug.ui/ui/org/eclipse/debug/internal/ui/views/console/ProcessConsoleManager.java
+++ b/org.eclipse.debug.ui/ui/org/eclipse/debug/internal/ui/views/console/ProcessConsoleManager.java
@@ -22,7 +22,11 @@ import java.util.Map;
 import org.eclipse.core.runtime.CoreException;
 import org.eclipse.core.runtime.IConfigurationElement;
 import org.eclipse.core.runtime.IExtensionPoint;
+import org.eclipse.core.runtime.IProgressMonitor;
+import org.eclipse.core.runtime.IStatus;
 import org.eclipse.core.runtime.Platform;
+import org.eclipse.core.runtime.Status;
+import org.eclipse.core.runtime.jobs.Job;
 import org.eclipse.debug.core.DebugPlugin;
 import org.eclipse.debug.core.ILaunch;
 import org.eclipse.debug.core.ILaunchListener;
@@ -46,7 +50,54 @@ import com.ibm.icu.text.MessageFormat;
  */
 public class ProcessConsoleManager implements ILaunchListener {
 
-    /**
+	/**
+	 * Crates console for given process
+	 */
+	private final class ConsoleCreation extends Job {
+		private final ILaunch launch;
+		private final IProcess process;
+
+		ConsoleCreation(ILaunch launch, IProcess process) {
+			super("Creating console for " + process.getLabel()); //$NON-NLS-1$
+			this.launch = launch;
+			this.process = process;
+			setSystem(true);
+			setUser(false);
+		}
+
+		@Override
+		protected IStatus run(IProgressMonitor monitor) {
+			if (monitor.isCanceled() || getConsoleDocument(process) != null) {
+				return Status.CANCEL_STATUS;
+			}
+			IConsoleColorProvider colorProvider = getColorProvider(process.getAttribute(IProcess.ATTR_PROCESS_TYPE));
+			String encoding = launch.getAttribute(DebugPlugin.ATTR_CONSOLE_ENCODING);
+			ProcessConsole pc = new ProcessConsole(process, colorProvider, encoding);
+			pc.setAttribute(IDebugUIConstants.ATTR_CONSOLE_PROCESS, process);
+
+			// add new console to console manager.
+			ConsolePlugin.getDefault().getConsoleManager().addConsoles(new IConsole[] { pc });
+			return Status.OK_STATUS;
+		}
+
+		@Override
+		public boolean belongsTo(Object family) {
+			return process == family || ProcessConsoleManager.class == family;
+		}
+
+		@Override
+		public boolean shouldSchedule() {
+			Job[] jobs = Job.getJobManager().find(process);
+			for (Job job : jobs) {
+				if (job instanceof ConsoleCreation) {
+					return false;
+				}
+			}
+			return true;
+		}
+	}
+
+	/**
      * Console document content provider extensions, keyed by extension id
      */
 	private Map<String, IConfigurationElement> fColorProviders;
@@ -139,21 +190,14 @@ public class ProcessConsoleManager implements ILaunchListener {
     @Override
 	public void launchChanged(final ILaunch launch) {
         IProcess[] processes= launch.getProcesses();
-        for (int i= 0; i < processes.length; i++) {
-            if (getConsoleDocument(processes[i]) == null) {
-                IProcess process = processes[i];
-                if (process.getStreamsProxy() == null) {
-                    continue;
-                }
-
-                //create a new console.
-                IConsoleColorProvider colorProvider = getColorProvider(process.getAttribute(IProcess.ATTR_PROCESS_TYPE));
-                String encoding = launch.getAttribute(DebugPlugin.ATTR_CONSOLE_ENCODING);
-                ProcessConsole pc = new ProcessConsole(process, colorProvider, encoding);
-                pc.setAttribute(IDebugUIConstants.ATTR_CONSOLE_PROCESS, process);
-
-                //add new console to console manager.
-                ConsolePlugin.getDefault().getConsoleManager().addConsoles(new IConsole[]{pc});
+		for (IProcess process : processes) {
+			if (process.getStreamsProxy() == null) {
+				continue;
+			}
+			if (getConsoleDocument(process) == null) {
+				// create a new console in a separated thread, see bug 355011.
+				Job job = new ConsoleCreation(launch, process);
+				job.schedule();
             }
         }
 		List<IProcess> removed = getRemovedProcesses(launch);
@@ -196,6 +240,7 @@ public class ProcessConsoleManager implements ILaunchListener {
      * launch listener and kills all existing console documents.
      */
     public void shutdown() {
+		Job.getJobManager().cancel(ProcessConsoleManager.class);
         ILaunchManager launchManager= DebugPlugin.getDefault().getLaunchManager();
         ILaunch[] launches = launchManager.getLaunches();
         for (int i = 0; i < launches.length; i++) {
-- 
cgit v1.2.3