Bug 545769 - [console] UTF-8 content read or send to process can beI20190517-1800I20190516-1800

corrupted

With some bad luck the ProcessConsole may disrupt multibyte UTF-8
characters read from input source (usually user input or file) due to
incorrect stream reading / buffer handling.
The same problem exist for reading the processes output streams.

Change-Id: I8d52d1973f3739e2c510a8a4c48b44f345c33dfe
Signed-off-by: Paul Pazderski <paul-eclipse@ppazderski.de>

3 files changed, 148 insertions, 1 deletions

diff --git a/org.eclipse.debug.tests/src/org/eclipse/debug/tests/AutomatedSuite.java b/org.eclipse.debug.tests/src/org/eclipse/debug/tests/AutomatedSuite.java
index 36beec028..d1858f09e 100644
--- a/org.eclipse.debug.tests/src/org/eclipse/debug/tests/AutomatedSuite.java
+++ b/org.eclipse.debug.tests/src/org/eclipse/debug/tests/AutomatedSuite.java
@@ -22,6 +22,7 @@ import org.eclipse.debug.tests.console.ConsoleTests;
 import org.eclipse.debug.tests.console.IOConsoleTests;
 import org.eclipse.debug.tests.console.ProcessConsoleManagerTests;
 import org.eclipse.debug.tests.console.ProcessConsoleTests;
+import org.eclipse.debug.tests.console.StreamsProxyTests;
 import org.eclipse.debug.tests.launching.AcceleratorSubstitutionTests;
 import org.eclipse.debug.tests.launching.ArgumentParsingTests;
 import org.eclipse.debug.tests.launching.LaunchConfigurationTests;
@@ -115,8 +116,9 @@ public class AutomatedSuite extends TestSuite {
 		addTest(new TestSuite(ConsoleManagerTests.class));
 		addTest(new TestSuite(ConsoleTests.class));
 		addTest(new TestSuite(IOConsoleTests.class));
-		addTest(new TestSuite(ProcessConsoleTests.class));
 		addTest(new TestSuite(ProcessConsoleManagerTests.class));
+		addTest(new TestSuite(ProcessConsoleTests.class));
+		addTest(new TestSuite(StreamsProxyTests.class));
 
 		// Launch Groups
 		addTest(new TestSuite(LaunchGroupTests.class));
diff --git a/org.eclipse.debug.tests/src/org/eclipse/debug/tests/console/ProcessConsoleTests.java b/org.eclipse.debug.tests/src/org/eclipse/debug/tests/console/ProcessConsoleTests.java
index 3ac3fa989..bdcc9e618 100644
--- a/org.eclipse.debug.tests/src/org/eclipse/debug/tests/console/ProcessConsoleTests.java
+++ b/org.eclipse.debug.tests/src/org/eclipse/debug/tests/console/ProcessConsoleTests.java
@@ -13,12 +13,18 @@
  *******************************************************************************/
 package org.eclipse.debug.tests.console;
 
+import java.nio.charset.StandardCharsets;
+import java.util.Collections;
+import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
 
 import org.eclipse.core.runtime.ILogListener;
 import org.eclipse.core.runtime.IStatus;
 import org.eclipse.core.runtime.Platform;
 import org.eclipse.core.runtime.jobs.Job;
+import org.eclipse.debug.core.DebugPlugin;
+import org.eclipse.debug.core.ILaunchManager;
+import org.eclipse.debug.core.Launch;
 import org.eclipse.debug.core.model.IProcess;
 import org.eclipse.debug.tests.AbstractDebugTest;
 import org.eclipse.debug.tests.TestUtil;
@@ -68,6 +74,68 @@ public class ProcessConsoleTests extends AbstractDebugTest {
 	}
 
 	/**
+	 * Test if two byte UTF-8 characters get disrupted on there way from process
+	 * console to the runtime process.
+	 * <p>
+	 * This test starts every two byte character on an even byte offset.
+	 * </p>
+	 */
+	public void testUTF8InputEven() throws Exception {
+		// 5000 characters result in 10000 bytes which should be more than most
+		// common buffer sizes.
+		processConsoleUTF8Input("", 5000);
+	}
+
+	/**
+	 * Test if two byte UTF-8 characters get disrupted on there way from process
+	 * console to the runtime process.
+	 * <p>
+	 * This test starts every two byte character on an odd byte offset.
+	 * </p>
+	 */
+	public void testUTF8InputOdd() throws Exception {
+		// 5000 characters result in 10000 bytes which should be more than most
+		// common buffer sizes.
+		processConsoleUTF8Input("+", 5000);
+	}
+
+	/**
+	 * Shared code for the UTF-8 input tests.
+	 * <p>
+	 * Send some two byte UTF-8 characters through process console user input
+	 * stream to mockup process and check if the input got corrupted on its way.
+	 * </p>
+	 *
+	 * @param prefix an arbitrary prefix inserted before the two byte UTF-8
+	 *            characters. Used to move the other characters to specific
+	 *            offsets e.g. a prefix of one byte will produce an input string
+	 *            where every two byte character starts at an odd offset.
+	 * @param numTwoByteCharacters number of two byte UTF-8 characters to send
+	 *            to process
+	 */
+	public void processConsoleUTF8Input(String prefix, int numTwoByteCharacters) throws Exception {
+		final String input = prefix + String.join("", Collections.nCopies(numTwoByteCharacters, "\u00F8"));
+		final MockProcess mockProcess = new MockProcess(input.getBytes(StandardCharsets.UTF_8).length, testTimeout);
+		try {
+			final IProcess process = DebugPlugin.newProcess(new Launch(null, ILaunchManager.RUN_MODE, null), mockProcess, "testUtf8Input");
+			@SuppressWarnings("restriction")
+			final org.eclipse.debug.internal.ui.views.console.ProcessConsole console = new org.eclipse.debug.internal.ui.views.console.ProcessConsole(process, new ConsoleColorProvider());
+			try {
+				console.initialize();
+				console.getInputStream().appendData(input);
+				mockProcess.waitFor(testTimeout, TimeUnit.MILLISECONDS);
+			} finally {
+				console.destroy();
+			}
+		} finally {
+			mockProcess.destroy();
+		}
+
+		final String receivedInput = new String(mockProcess.getReceivedInput(), StandardCharsets.UTF_8);
+		assertEquals(input, receivedInput);
+	}
+
+	/**
 	 * Test if InputReadJob can be canceled.
 	 * <p>
 	 * Actually tests cancellation for all jobs of
diff --git a/org.eclipse.debug.tests/src/org/eclipse/debug/tests/console/StreamsProxyTests.java b/org.eclipse.debug.tests/src/org/eclipse/debug/tests/console/StreamsProxyTests.java
new file mode 100644
index 000000000..b4ffdb4a4
--- /dev/null
+++ b/org.eclipse.debug.tests/src/org/eclipse/debug/tests/console/StreamsProxyTests.java
@@ -0,0 +1,77 @@
+/*******************************************************************************
+ * Copyright (c) 2019 Paul Pazderski and others.
+ *
+ * This program and the accompanying materials
+ * are made available under the terms of the Eclipse Public License 2.0
+ * which accompanies this distribution, and is available at
+ * https://www.eclipse.org/legal/epl-2.0/
+ *
+ * SPDX-License-Identifier: EPL-2.0
+ *
+ * Contributors:
+ *     Paul Pazderski - initial API and implementation
+ *******************************************************************************/
+package org.eclipse.debug.tests.console;
+
+import java.io.ByteArrayInputStream;
+import java.nio.charset.StandardCharsets;
+import java.util.Collections;
+
+import org.eclipse.debug.internal.core.StreamsProxy;
+import org.eclipse.debug.tests.AbstractDebugTest;
+
+/**
+ * Tests the {@link StreamsProxy}.
+ */
+public class StreamsProxyTests extends AbstractDebugTest {
+
+	public StreamsProxyTests() {
+		super(StreamsProxyTests.class.getSimpleName());
+	}
+
+	public StreamsProxyTests(String name) {
+		super(name);
+	}
+
+	/**
+	 * Test console receiving UTF-8 output from process where two-byte UTF-8
+	 * characters start at even offsets.
+	 */
+	public void testReceiveUTF8Even() throws Exception {
+		// 4500 characters results in 9000 byte of output which should be more
+		// than most common buffer sizes.
+		receiveUTF8Test("", 4500);
+	}
+
+	/**
+	 * Test console receiving UTF-8 output from process where two-byte UTF-8
+	 * characters start at odd offsets.
+	 */
+	public void testReceiveUTF8Odd() throws Exception {
+		// 4500 characters results in 9000 byte of output which should be more
+		// than most common buffer sizes.
+		receiveUTF8Test("+", 4500);
+	}
+
+	/**
+	 * Shared code for the UTF-8 tests.
+	 * <p>
+	 * Receive some UTF-8 content from a (mockup) process output stream.
+	 * </p>
+	 *
+	 * @param prefix an arbitrary prefix inserted before the two byte UTF-8
+	 *            characters. Used to move the other characters to specific
+	 *            offsets e.g. a prefix of one byte will produce output where
+	 *            every two byte character starts at an odd offset.
+	 * @param numTwoByteCharacters number of two byte UTF-8 characters to output
+	 */
+	private void receiveUTF8Test(String prefix, int numTwoByteCharacters) throws Exception {
+		final String s = prefix + String.join("", Collections.nCopies(numTwoByteCharacters, "\u00F8"));
+		final ByteArrayInputStream stdout = new ByteArrayInputStream(s.getBytes(StandardCharsets.UTF_8));
+		final Process mockProcess = new MockProcess(stdout, null, 0);
+		final StreamsProxy streamProxy = new StreamsProxy(mockProcess, StandardCharsets.UTF_8.name());
+		streamProxy.close();
+		final String readContent = streamProxy.getOutputStreamMonitor().getContents();
+		assertEquals("Process output got corrupted.", s, readContent);
+	}
+}