Enhanced control over master sdm to debug parallel applications. Also added tracing to sdm plugin.

7 files changed, 346 insertions, 158 deletions

diff --git a/debug/org.eclipse.ptp.debug.core/src/org/eclipse/ptp/debug/core/IPDebugger.java b/debug/org.eclipse.ptp.debug.core/src/org/eclipse/ptp/debug/core/IPDebugger.java
index c3e336d92..000fcde75 100644
--- a/debug/org.eclipse.ptp.debug.core/src/org/eclipse/ptp/debug/core/IPDebugger.java
+++ b/debug/org.eclipse.ptp.debug.core/src/org/eclipse/ptp/debug/core/IPDebugger.java
@@ -69,6 +69,5 @@ public interface IPDebugger {
 	 */
 	public void getLaunchAttributes(ILaunchConfiguration configuration, AttributeManager attrMgr) throws CoreException;
 
-//	public void prepare(ILaunchConfiguration configuration, AttributeManager attrMgr);
-//	public void cleanup(ILaunchConfiguration configuration, AttributeManager attrMgr, IPLaunch launch);
+	public void cleanup(ILaunchConfiguration configuration, AttributeManager attrMgr, IPLaunch launch);
 }
diff --git a/debug/org.eclipse.ptp.debug.sdm.core/.options b/debug/org.eclipse.ptp.debug.sdm.core/.options
new file mode 100644
index 000000000..4214750a9
--- /dev/null
+++ b/debug/org.eclipse.ptp.debug.sdm.core/.options
@@ -0,0 +1,5 @@
+# Trace job that executes SDM master
+org.eclipse.ptp.debug.sdm.core/debug=true
+org.eclipse.ptp.debug.sdm.core/debug/SDMmaster=true
+org.eclipse.ptp.debug.sdm.core/debug/SDMmaster/more=true
+org.eclipse.ptp.debug.sdm.core/debug/SDMmaster/output=true
diff --git a/debug/org.eclipse.ptp.debug.sdm.core/META-INF/MANIFEST.MF b/debug/org.eclipse.ptp.debug.sdm.core/META-INF/MANIFEST.MF
index c27c60559..cd9880bdb 100644
--- a/debug/org.eclipse.ptp.debug.sdm.core/META-INF/MANIFEST.MF
+++ b/debug/org.eclipse.ptp.debug.sdm.core/META-INF/MANIFEST.MF
@@ -39,6 +39,7 @@ Require-Bundle: org.eclipse.core.runtime,
  org.eclipse.ptp.launch,
  org.eclipse.ptp.remote.core,
  org.eclipse.ptp.rm.remote,
- org.eclipse.core.filesystem;bundle-version="1.2.0"
+ org.eclipse.core.filesystem;bundle-version="1.2.0",
+ org.eclipse.search;bundle-version="3.4.0"
 Eclipse-LazyStart: true
 Bundle-Vendor: Eclipse.org
diff --git a/debug/org.eclipse.ptp.debug.sdm.core/src/org/eclipse/ptp/debug/sdm/core/SDMDebugCorePlugin.java b/debug/org.eclipse.ptp.debug.sdm.core/src/org/eclipse/ptp/debug/sdm/core/SDMDebugCorePlugin.java
index 251fe3fd6..9ac0d326c 100644
--- a/debug/org.eclipse.ptp.debug.sdm.core/src/org/eclipse/ptp/debug/sdm/core/SDMDebugCorePlugin.java
+++ b/debug/org.eclipse.ptp.debug.sdm.core/src/org/eclipse/ptp/debug/sdm/core/SDMDebugCorePlugin.java
@@ -1,6 +1,7 @@
 package org.eclipse.ptp.debug.sdm.core;
 
 import org.eclipse.core.runtime.Plugin;
+import org.eclipse.ptp.debug.sdm.core.utils.DebugUtil;
 import org.osgi.framework.BundleContext;
 
 /**
@@ -20,7 +21,7 @@ public class SDMDebugCorePlugin extends Plugin {
 
 	/**
 	 * Get a unique identifier for this plugin
-	 * 
+	 *
 	 * @return
 	 */
 	public static String getUniqueIdentifier() {
@@ -38,13 +39,16 @@ public class SDMDebugCorePlugin extends Plugin {
 	/**
 	 * This method is called upon plug-in activation
 	 */
+	@Override
 	public void start(BundleContext context) throws Exception {
 		super.start(context);
+		DebugUtil.configurePluginDebugOptions();
 	}
-	
+
 	/**
 	 * This method is called when the plug-in is stopped
 	 */
+	@Override
 	public void stop(BundleContext context) throws Exception {
 		super.stop(context);
 		plugin = null;
diff --git a/debug/org.eclipse.ptp.debug.sdm.core/src/org/eclipse/ptp/debug/sdm/core/SDMDebugger.java b/debug/org.eclipse.ptp.debug.sdm.core/src/org/eclipse/ptp/debug/sdm/core/SDMDebugger.java
index 0aa369bb0..741984fbc 100644
--- a/debug/org.eclipse.ptp.debug.sdm.core/src/org/eclipse/ptp/debug/sdm/core/SDMDebugger.java
+++ b/debug/org.eclipse.ptp.debug.sdm.core/src/org/eclipse/ptp/debug/sdm/core/SDMDebugger.java
@@ -18,9 +18,7 @@
  *******************************************************************************/
 package org.eclipse.ptp.debug.sdm.core;
 
-import java.io.BufferedReader;
 import java.io.IOException;
-import java.io.InputStreamReader;
 import java.io.OutputStream;
 import java.io.PrintWriter;
 import java.util.ArrayList;
@@ -40,7 +38,6 @@ import org.eclipse.core.runtime.Path;
 import org.eclipse.core.runtime.Preferences;
 import org.eclipse.core.runtime.Status;
 import org.eclipse.debug.core.ILaunchConfiguration;
-import org.eclipse.osgi.util.NLS;
 import org.eclipse.ptp.core.IPTPLaunchConfigurationConstants;
 import org.eclipse.ptp.core.PTPCorePlugin;
 import org.eclipse.ptp.core.attributes.ArrayAttribute;
@@ -66,13 +63,13 @@ import org.eclipse.ptp.debug.core.pdi.event.IPDIEventFactory;
 import org.eclipse.ptp.debug.core.pdi.manager.IPDIManagerFactory;
 import org.eclipse.ptp.debug.core.pdi.model.IPDIModelFactory;
 import org.eclipse.ptp.debug.core.pdi.request.IPDIRequestFactory;
+import org.eclipse.ptp.debug.sdm.core.SDMRunner.SDMMasterState;
 import org.eclipse.ptp.debug.sdm.core.pdi.PDIDebugger;
+import org.eclipse.ptp.debug.sdm.core.utils.DebugUtil;
 import org.eclipse.ptp.launch.PTPLaunchPlugin;
 import org.eclipse.ptp.remote.core.IRemoteConnection;
 import org.eclipse.ptp.remote.core.IRemoteConnectionManager;
 import org.eclipse.ptp.remote.core.IRemoteFileManager;
-import org.eclipse.ptp.remote.core.IRemoteProcess;
-import org.eclipse.ptp.remote.core.IRemoteProcessBuilder;
 import org.eclipse.ptp.remote.core.IRemoteServices;
 import org.eclipse.ptp.remote.core.PTPRemoteCorePlugin;
 import org.eclipse.ptp.rm.remote.core.AbstractRemoteResourceManagerConfiguration;
@@ -90,50 +87,7 @@ public class SDMDebugger implements IPDebugger {
 	private IPDIRequestFactory requestFactory = null;
 
 	IFileStore routingFileStore = null;
-	private List<String> dbgArgs;
-	private IRemoteProcess sdmProcess = null;
-	private IRemoteProcessBuilder sdmProcessBuilder = null;
-	
-	private class SDMStarter extends Thread {
-		private CoreException e;
-		private IPLaunch launch;
-		
-		public SDMStarter(IPLaunch launch) {
-			super();
-			this.launch = launch;
-		}
-
-		@Override
-		public void run() {
-			synchronized (this) {
-				try {
-					wait(3000);
-					startMasterSDM(launch);
-					
-					/*
-					 * Check if process has completed early (failed)
-					 */
-					wait(100);
-					if (sdmProcess.isCompleted()) {
-						throw new CoreException(new Status(IStatus.ERROR, SDMDebugCorePlugin.getUniqueIdentifier(), NLS.bind("Master SDM process finished early with exit code {0}.", sdmProcess.exitValue()), e));
-					}
-				} catch (InterruptedException e) {
-					stopMasterSDM();
-					this.e = new CoreException(new Status(IStatus.ERROR, SDMDebugCorePlugin.getUniqueIdentifier(), "Start of master SDM was interrupted", e));
-				} catch (CoreException e) {
-					this.e = new CoreException(new Status(IStatus.ERROR, SDMDebugCorePlugin.getUniqueIdentifier(), "Failed to create master SDM process", e));
-				}
-			}
-		}
-				
-		/**
-		 * Gets the exception that describes why SDM master failed to start.
-		 * @return A CoreException if SDM failed to start of null if SDM was successful.
-		 */
-		public CoreException getCoreException() {
-			return e;
-		}
-	}
+	SDMRunner sdmRunner = null;
 
 	/* (non-Javadoc)
 	 * @see org.eclipse.ptp.debug.core.IPDebugger#createDebugSession(long, org.eclipse.ptp.debug.core.launch.IPLaunch, org.eclipse.core.runtime.IPath)
@@ -151,29 +105,21 @@ public class SDMDebugger implements IPDebugger {
 		if (requestFactory == null) {
 			requestFactory = new SDMRequestFactory();
 		}
-		
+
 		/*
 		 * Writing the rounting file actually starts the SDM servers.
 		 */
 		writeRoutingFile(launch);
 
 		/*
-		 * Delay starting the master SDM (aka SDM client), to wait intil SDM servers have started and until the sessions
-		 * is listening on the debugger socket.
+		 * Delay starting the master SDM (aka SDM client), to wait until SDM servers have started and until the sessions
+		 * are listening on the debugger socket.
 		 */
-		SDMStarter sdmStarter = new SDMStarter(launch);
-		sdmStarter.start();
+		sdmRunner.setJob(launch.getPJob());
+		sdmRunner.schedule();
+
 		IPDISession session = createSession(timeout, launch, corefile);
-		while (sdmStarter.isAlive()) {
-			try {
-				sdmStarter.join();
-			} catch (InterruptedException e) {
-				// Ignore and continue waiting.
-			}
-		}
-		if (sdmStarter.getCoreException() != null) {
-			throw sdmStarter.getCoreException();
-		}
+
 		return session;
 	}
 
@@ -196,7 +142,17 @@ public class SDMDebugger implements IPDebugger {
 			throw newCoreException(e);
 		}
 
+		/*
+		 * Store information to create routing file later.
+		 */
 		prepareRoutingFile(configuration, attrMgr, monitor);
+
+		/*
+		 * Prepare the Master SDM controller thread.
+		 */
+		IResourceManagerControl rm = null;
+		rm = (IResourceManagerControl) getResourceManager(configuration);
+		sdmRunner = new SDMRunner(rm);
 	}
 
 	/* (non-Javadoc)
@@ -210,7 +166,7 @@ public class SDMDebugger implements IPDebugger {
 			attrMgr.addAttribute(dbgArgsAttr);
 		}
 
-		dbgArgs = dbgArgsAttr.getValue();
+		List<String> dbgArgs = dbgArgsAttr.getValue();
 
 		Preferences store = SDMDebugCorePlugin.getDefault().getPluginPreferences();
 
@@ -256,17 +212,44 @@ public class SDMDebugger implements IPDebugger {
 			attrMgr.addAttribute(JobAttributes.getExecutablePathAttributeDefinition().create(dbgWD + "/Debug")); //$NON-NLS-1$
 		}
 		attrMgr.addAttribute(JobAttributes.getDebugFlagAttributeDefinition().create(true));
-		
-		prepareMasterSDM(configuration, attrMgr, new NullProgressMonitor());
+
+		/*
+		 * Save SDM command line for future use.
+		 */
+		List<String> sdmCommand = new ArrayList<String>();
+		sdmCommand.add(attrMgr.getAttribute(JobAttributes.getDebuggerExecutablePathAttributeDefinition()).getValue()+"/"+attrMgr.getAttribute(JobAttributes.getDebuggerExecutableNameAttributeDefinition()).getValue());
+		sdmCommand.addAll(dbgArgs);
+		sdmRunner.setCommand(sdmCommand);
+		sdmRunner.setWorkDir(attrMgr.getAttribute(JobAttributes.getWorkingDirectoryAttributeDefinition()).getValue());
 	}
 
-//	public void cleanup(ILaunchConfiguration configuration,
-//			AttributeManager attrMgr, IPLaunch launch) {
-//		if (process != null) {
-//			process.destroy();
-//			process = null;
-//		}
-//	}
+	public void cleanup(ILaunchConfiguration configuration, AttributeManager attrMgr, IPLaunch launch) {
+		if (sdmRunner != null) {
+			if (sdmRunner.getSdmState() == SDMMasterState.RUNNING) {
+				DebugUtil.trace(DebugUtil.SDM_MASTER_TRACING, "sdm master: still running, cancel is to be issued soon"); //$NON-NLS-1$
+				new Thread("SDM master killer thread") {
+					@Override
+					public void run() {
+						DebugUtil.trace(DebugUtil.SDM_MASTER_TRACING_MORE, "sdm master killer: thread started"); //$NON-NLS-1$
+						synchronized (this) {
+							try {
+								wait(5000);
+							} catch (InterruptedException e) {
+								// Ignore
+							}
+						}
+						if (sdmRunner.getSdmState() == SDMMasterState.RUNNING) {
+							DebugUtil.trace(DebugUtil.SDM_MASTER_TRACING, "sdm master killer: cancel SDM master now"); //$NON-NLS-1$
+							sdmRunner.cancel();
+						} else {
+							DebugUtil.trace(DebugUtil.SDM_MASTER_TRACING, "sdm master killer: do not cancel SDM master, since it finished by itself."); //$NON-NLS-1$
+						}
+						DebugUtil.trace(DebugUtil.SDM_MASTER_TRACING_MORE, "sdm master killer: thread finished"); //$NON-NLS-1$
+					}
+				}.start();
+			}
+		}
+	}
 
 	/**
 	 * Get the PDI debugger implementation. Creates the class if necessary.
@@ -402,83 +385,4 @@ public class SDMDebugger implements IPDebugger {
 			throw newCoreException(e);
 		}
 	}
-	
-	private void prepareMasterSDM(ILaunchConfiguration configuration,
-			AttributeManager attrMgr, IProgressMonitor monitor) throws CoreException {
-		IResourceManagerControl rm = null;
-		
-		List<String> cmd = new ArrayList<String>();
-		cmd.add(attrMgr.getAttribute(JobAttributes.getDebuggerExecutablePathAttributeDefinition()).getValue()+"/"+attrMgr.getAttribute(JobAttributes.getDebuggerExecutableNameAttributeDefinition()).getValue());
-		cmd.addAll(dbgArgs);
-
-		try {
-			rm = (IResourceManagerControl) getResourceManager(configuration);
-		} catch (CoreException e) {
-			e.printStackTrace();
-		}
-		IResourceManagerConfiguration conf = rm.getConfiguration();
-		AbstractRemoteResourceManagerConfiguration remConf = (AbstractRemoteResourceManagerConfiguration)conf;
-		IRemoteServices remoteServices = PTPRemoteCorePlugin.getDefault().getRemoteServices(remConf.getRemoteServicesId());
-		IRemoteConnectionManager connectionManager = remoteServices.getConnectionManager();
-		IRemoteConnection connection = connectionManager.getConnection(remConf.getConnectionName());
-		sdmProcessBuilder = remoteServices.getProcessBuilder(connection, cmd);
-		String workdir = attrMgr.getAttribute(JobAttributes.getWorkingDirectoryAttributeDefinition()).getValue();
-		IRemoteFileManager fileManager = remoteServices.getFileManager(connection);
-		IFileStore directory = null;
-		try {
-			directory = fileManager.getResource(new Path(workdir), monitor);
-		} catch (IOException e) {
-			throw newCoreException(e);
-		}
-		sdmProcessBuilder.directory(directory);
-	}
-	
-	private void startMasterSDM(IPLaunch launch) throws CoreException {
-		assert sdmProcess == null;
-		assert sdmProcessBuilder != null;
-		try {
-			sdmProcess = sdmProcessBuilder.start();
-		} catch (IOException e) {
-			throw newCoreException(e);
-		}
-		
-		final BufferedReader err_reader = new BufferedReader(new InputStreamReader(sdmProcess.getErrorStream()));
-		final BufferedReader out_reader = new BufferedReader(new InputStreamReader(sdmProcess.getInputStream()));
-
-		new Thread(new Runnable() {
-			public void run() {
-				try {
-					String output;
-					while ((output = out_reader.readLine()) != null) {
-						System.out.println("sdm master: " + output); //$NON-NLS-1$
-					}
-				} catch (IOException e) {
-					// Ignore
-				}
-			}
-		}, "SDM master standard output thread").start(); //$NON-NLS-1$
-		
-		new Thread(new Runnable() {
-			public void run() {
-				try {
-					String line;
-					while ((line = err_reader.readLine()) != null) {
-						System.err.println("sdm master: " + line); //$NON-NLS-1$
-					}
-				} catch (IOException e) {
-					// Ignore
-				}
-			}
-		}, "SDM master error output thread").start(); //$NON-NLS-1$
-
-		sdmProcessBuilder = null;
-	}
-	
-	public void stopMasterSDM() {
-		if (sdmProcess != null) {
-			sdmProcess.destroy();
-			sdmProcess = null;
-		}
-	}
-
 }
diff --git a/debug/org.eclipse.ptp.debug.sdm.core/src/org/eclipse/ptp/debug/sdm/core/SDMRunner.java b/debug/org.eclipse.ptp.debug.sdm.core/src/org/eclipse/ptp/debug/sdm/core/SDMRunner.java
new file mode 100644
index 000000000..836662e31
--- /dev/null
+++ b/debug/org.eclipse.ptp.debug.sdm.core/src/org/eclipse/ptp/debug/sdm/core/SDMRunner.java
@@ -0,0 +1,220 @@
+package org.eclipse.ptp.debug.sdm.core;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.util.List;
+
+import org.eclipse.core.filesystem.IFileStore;
+import org.eclipse.core.runtime.CoreException;
+import org.eclipse.core.runtime.IProgressMonitor;
+import org.eclipse.core.runtime.IStatus;
+import org.eclipse.core.runtime.Path;
+import org.eclipse.core.runtime.Status;
+import org.eclipse.core.runtime.jobs.Job;
+import org.eclipse.osgi.util.NLS;
+import org.eclipse.ptp.core.elementcontrols.IResourceManagerControl;
+import org.eclipse.ptp.core.elements.IPJob;
+import org.eclipse.ptp.debug.core.PTPDebugCorePlugin;
+import org.eclipse.ptp.debug.sdm.core.utils.DebugUtil;
+import org.eclipse.ptp.remote.core.IRemoteConnection;
+import org.eclipse.ptp.remote.core.IRemoteConnectionManager;
+import org.eclipse.ptp.remote.core.IRemoteFileManager;
+import org.eclipse.ptp.remote.core.IRemoteProcess;
+import org.eclipse.ptp.remote.core.IRemoteProcessBuilder;
+import org.eclipse.ptp.remote.core.IRemoteServices;
+import org.eclipse.ptp.remote.core.PTPRemoteCorePlugin;
+import org.eclipse.ptp.rm.remote.core.AbstractRemoteResourceManagerConfiguration;
+
+public class SDMRunner extends Job {
+	public enum SDMMasterState {UNKNOWN, STARTING, RUNNING, FINISHED, ERROR};
+
+	private List<String> command;
+	private String workDir;
+	private SDMMasterState sdmState = SDMMasterState.STARTING;
+
+	private IPJob ipJob = null;
+	private IResourceManagerControl rmControl = null;
+	private IRemoteProcess sdmProcess;
+
+	public SDMRunner(IResourceManagerControl rmControl) {
+		super("Master SDM control");
+		this.setPriority(Job.LONG);
+		this.setSystem(true);
+		this.setSystem(false);
+		this.rmControl = rmControl;
+		DebugUtil.trace(DebugUtil.SDM_MASTER_TRACING_MORE, "Created sdm master runner."); //$NON-NLS-1$
+	}
+
+	public void setCommand(List<String> command) {
+		DebugUtil.trace(DebugUtil.SDM_MASTER_TRACING_MORE, "sdm master: command: {0}", command.toString()); //$NON-NLS-1$
+		this.command = command;
+	}
+
+	public void setWorkDir(String workDir) {
+		DebugUtil.trace(DebugUtil.SDM_MASTER_TRACING_MORE, "sdm master: workdir: {0}", workDir); //$NON-NLS-1$
+		this.workDir = workDir;
+	}
+
+	public synchronized SDMMasterState getSdmState() {
+		return sdmState;
+	}
+
+	protected synchronized void setSdmState(SDMMasterState sdmState) {
+		DebugUtil.trace(DebugUtil.SDM_MASTER_TRACING_MORE, "sdm master: changed state to {0}", sdmState.toString()); //$NON-NLS-1$
+		this.sdmState = sdmState;
+		this.notifyAll();
+	}
+
+	public void setJob(IPJob ipJob) {
+		DebugUtil.trace(DebugUtil.SDM_MASTER_TRACING_MORE, "sdm master: associated to job #{0}", ipJob.getID()); //$NON-NLS-1$
+		this.ipJob = ipJob;
+	}
+
+	@Override
+	protected IStatus run(IProgressMonitor monitor) {
+		assert workDir != null;
+		assert command != null;
+		assert sdmProcess == null;
+
+		DebugUtil.trace(DebugUtil.SDM_MASTER_TRACING, "sdm master: thread started."); //$NON-NLS-1$
+		/*
+		 * Catch all try...catch
+		 */
+		try {
+			if (monitor.isCanceled()) throw new InterruptedException();
+			/*
+			 * Prepare remote connection.
+			 */
+			AbstractRemoteResourceManagerConfiguration configuration = (AbstractRemoteResourceManagerConfiguration) rmControl.getConfiguration();
+			IRemoteServices remoteServices = PTPRemoteCorePlugin.getDefault().getRemoteServices(configuration.getRemoteServicesId());
+			IRemoteConnectionManager connectionManager = remoteServices.getConnectionManager();
+			IRemoteConnection connection = connectionManager.getConnection(configuration.getConnectionName());
+			IRemoteFileManager fileManager = remoteServices.getFileManager(connection);
+
+			IFileStore directory = null;
+			try {
+				directory = fileManager.getResource(new Path(workDir), monitor);
+			} catch (IOException e) {
+				throw new CoreException(new Status(IStatus.ERROR, PTPDebugCorePlugin.getUniqueIdentifier(), "Failed to determine working directory for master SDM.", e));
+			}
+			IRemoteProcessBuilder sdmProcessBuilder = remoteServices.getProcessBuilder(connection, command);
+			sdmProcessBuilder.directory(directory);
+
+			/*
+			 * Wait some time to assure that SDM servers and front end have started.
+			 */
+			DebugUtil.trace(DebugUtil.SDM_MASTER_TRACING, "sdm master: waiting for SDM servers and front end."); //$NON-NLS-1$
+			if (monitor.isCanceled()) throw new InterruptedException();
+			synchronized (this) {
+				wait(3000);
+			}
+
+			/*
+			 * Create process.
+			 */
+			DebugUtil.trace(DebugUtil.SDM_MASTER_TRACING, "sdm master: create process"); //$NON-NLS-1$
+			if (monitor.isCanceled()) throw new InterruptedException();
+			synchronized (this) {
+				sdmProcess = sdmProcessBuilder.start();
+			}
+			final BufferedReader err_reader = new BufferedReader(new InputStreamReader(sdmProcess.getErrorStream()));
+			final BufferedReader out_reader = new BufferedReader(new InputStreamReader(sdmProcess.getInputStream()));
+
+			if (DebugUtil.SDM_MASTER_OUTPUT_TRACING) {
+				new Thread(new Runnable() {
+					public void run() {
+						try {
+							String output;
+							while ((output = out_reader.readLine()) != null) {
+								System.out.println("sdm master: " + output); //$NON-NLS-1$
+							}
+						} catch (IOException e) {
+							// Ignore
+						}
+					}
+				}, "sdm master standard output thread").start(); //$NON-NLS-1$
+			}
+
+			if (DebugUtil.SDM_MASTER_OUTPUT_TRACING) {
+				new Thread(new Runnable() {
+					public void run() {
+						try {
+							String line;
+							while ((line = err_reader.readLine()) != null) {
+								System.err.println("sdm master: " + line); //$NON-NLS-1$
+							}
+						} catch (IOException e) {
+							// Ignore
+						}
+					}
+				}, "sdm master error output thread").start(); //$NON-NLS-1$
+			}
+
+
+			/*
+			 * Wait while running but not canceled.
+			 */
+			setSdmState(SDMMasterState.RUNNING);
+			DebugUtil.trace(DebugUtil.SDM_MASTER_TRACING_MORE, "sdm master: waiting to finish."); //$NON-NLS-1$
+			while (! sdmProcess.isCompleted()) {
+				synchronized (this) {
+					wait(500);
+				}
+				if (monitor.isCanceled()) throw new InterruptedException();
+			}
+
+			/*
+			 * Check if process terminated successfully (if not canceled).
+			 */
+			DebugUtil.trace(DebugUtil.SDM_MASTER_TRACING, "sdm master: exit value {0}.", sdmProcess.exitValue()); //$NON-NLS-1$
+			if (sdmProcess.exitValue() != 0) {
+				if (! monitor.isCanceled()) {
+					throw new CoreException(new Status(IStatus.ERROR, SDMDebugCorePlugin.getUniqueIdentifier(), NLS.bind("sdm master process finished with exit code {0}.", sdmProcess.exitValue())));
+				} else {
+					DebugUtil.trace(DebugUtil.SDM_MASTER_TRACING, "sdm master: ignoring exit code since requested to terminate."); //$NON-NLS-1$
+				}
+			}
+			setSdmState(SDMMasterState.FINISHED);
+			return Status.OK_STATUS;
+		} catch (Exception e) {
+			/*
+			 * Terminate the job, handling the error.
+			 * Also terminates the ipjob since it does not make sense to the ipjob running without debugger.
+			 */
+			DebugUtil.error(DebugUtil.SDM_MASTER_TRACING, "sdm master: exception: {0}", e); //$NON-NLS-1$
+			setSdmState(SDMMasterState.ERROR);
+			synchronized (this) {
+				DebugUtil.error(DebugUtil.SDM_MASTER_TRACING, "sdm master: destroy process", e); //$NON-NLS-1$
+				sdmProcess.destroy();
+			}
+			try {
+				DebugUtil.trace(DebugUtil.SDM_MASTER_TRACING, "sdm master: request job {0} to terminate.", ipJob.getID()); //$NON-NLS-1$
+				rmControl.terminateJob(ipJob);
+			} catch (CoreException e1) {
+				PTPDebugCorePlugin.log(e1);
+			}
+			if (e instanceof InterruptedException) {
+				return Status.CANCEL_STATUS;
+			} else if (e instanceof CoreException) {
+				return ((CoreException) e).getStatus();
+			} else {
+				return new Status(IStatus.ERROR, SDMDebugCorePlugin.getUniqueIdentifier(), "Failed to launch sdm master process.", e);
+			}
+		} finally {
+			DebugUtil.trace(DebugUtil.SDM_MASTER_TRACING, "sdm master: thread finished."); //$NON-NLS-1$
+			synchronized (this) {
+				sdmProcess = null;
+			}
+		}
+	}
+
+	@Override
+	protected void canceling() {
+		synchronized (this) {
+			if (sdmProcess != null) {
+				sdmProcess.destroy();
+			}
+		}
+	}
+}
diff --git a/debug/org.eclipse.ptp.debug.sdm.core/src/org/eclipse/ptp/debug/sdm/core/utils/DebugUtil.java b/debug/org.eclipse.ptp.debug.sdm.core/src/org/eclipse/ptp/debug/sdm/core/utils/DebugUtil.java
new file mode 100644
index 000000000..e4a1635de
--- /dev/null
+++ b/debug/org.eclipse.ptp.debug.sdm.core/src/org/eclipse/ptp/debug/sdm/core/utils/DebugUtil.java
@@ -0,0 +1,55 @@
+package org.eclipse.ptp.debug.sdm.core.utils;
+
+
+import java.text.MessageFormat;
+
+import org.eclipse.core.runtime.Platform;
+import org.eclipse.ptp.debug.sdm.core.SDMDebugCorePlugin;
+
+public class DebugUtil {
+	private static final String SDM_MASTER_TRACING_OPTION = "org.eclipse.ptp.debug.sdm.core/debug/SDMmaster"; //$NON-NLS-1$
+	private static final String SDM_MASTER_TRACING_OPTION_MORE = "org.eclipse.ptp.debug.sdm.core/debug/SDMmaster/more"; //$NON-NLS-1$
+	private static final String SDM_MASTER_OUTPUT_TRACING_OPTION = "org.eclipse.ptp.debug.sdm.core/debug/SDMmaster/output"; //$NON-NLS-1$
+
+	public static boolean SDM_MASTER_TRACING = false;
+	public static boolean SDM_MASTER_TRACING_MORE = false;
+	public static boolean SDM_MASTER_OUTPUT_TRACING = false;
+
+	public static void configurePluginDebugOptions() {
+		if (SDMDebugCorePlugin.getDefault().isDebugging()) {
+			String option = Platform.getDebugOption(SDM_MASTER_TRACING_OPTION);
+			if (option != null) {
+				SDM_MASTER_TRACING = option.equalsIgnoreCase("true") ; //$NON-NLS-1$
+			}
+			option = Platform.getDebugOption(SDM_MASTER_TRACING_OPTION_MORE);
+			if (option != null) {
+				SDM_MASTER_TRACING_MORE = option.equalsIgnoreCase("true") ; //$NON-NLS-1$
+			}
+			option = Platform.getDebugOption(SDM_MASTER_OUTPUT_TRACING_OPTION);
+			if (option != null) {
+				SDM_MASTER_OUTPUT_TRACING = option.equalsIgnoreCase("true") ; //$NON-NLS-1$
+			}
+		}
+	}
+
+	public static void trace(boolean option, String pattern, Object ... arguments) {
+		trace(option, MessageFormat.format(pattern, arguments));
+	}
+
+	public static void trace(boolean option, String message) {
+		if (option) {
+			System.out.println(message);
+			System.out.flush();
+		}
+	}
+
+	public static void error(boolean option, String pattern, Object ... arguments) {
+		error(option, MessageFormat.format(pattern, arguments));
+	}
+
+	public static void error(boolean option, String message) {
+		if (option) {
+			System.err.println(message);
+		}
+	}
+}