Bug 545769 - [tests] Test for possible UTF-8 corruption from/to process

The ProcessConsole input forwarding may corrupt two byte UTF-8
characters. The same problem may exist for reading the process output.

Change-Id: I7bb60282d7d174de16ec9d7be318f80cfc733693
Signed-off-by: Paul Pazderski <paul-eclipse@ppazderski.de>

4 files changed, 207 insertions, 4 deletions

diff --git a/org.eclipse.jdt.debug.tests/testprograms/ConsoleOutputUmlaut.java b/org.eclipse.jdt.debug.tests/testprograms/ConsoleOutputUmlaut.java
new file mode 100644
index 000000000..01edfaa52
--- /dev/null
+++ b/org.eclipse.jdt.debug.tests/testprograms/ConsoleOutputUmlaut.java
@@ -0,0 +1,64 @@
+/*******************************************************************************
+ * Copyright (c) 2019 Paul Pazderski and others.
+ *
+ * This program and the accompanying materials
+ * are made available under the terms of the Eclipse Public License 2.0
+ * which accompanies this distribution, and is available at
+ * https://www.eclipse.org/legal/epl-2.0/
+ *
+ * SPDX-License-Identifier: EPL-2.0
+ *
+ * Contributors:
+ *     Paul Pazderski - initial API and implementation
+ *******************************************************************************/
+
+import java.nio.charset.Charset;
+
+/**
+ * This snippet prints a configurable number of one byte characters followed by a configurable number of two byte characters.
+ */
+public class ConsoleOutputUmlaut {
+	public static void main(String[] args) {
+		if (!"utf-8".equalsIgnoreCase(Charset.defaultCharset().name()) && !"utf8".equalsIgnoreCase(Charset.defaultCharset().name())) {
+			System.err.println("The programm's output must be UTF-8 encoded.");
+			System.exit(2);
+		}
+
+		int numAscii = 1;
+		int numUmlaut = 4200;
+		int repetitions = 1;
+
+		if (args.length > 0) {
+			try {
+				numAscii = Integer.parseInt(args[0]);
+			} catch (NumberFormatException e) {
+			}
+		}
+		if (args.length > 1) {
+			try {
+				numUmlaut = Integer.parseInt(args[1]);
+			} catch (NumberFormatException e) {
+			}
+		}
+		if (args.length > 2) {
+			try {
+				repetitions = Integer.parseInt(args[2]);
+			} catch (NumberFormatException e) {
+			}
+		}
+
+		StringBuilder sb = new StringBuilder(numAscii + numUmlaut + 2);
+		for (int i = 0; i < numAscii; i++) {
+			sb.append('0');
+		}
+		for (int i = 0; i < numUmlaut; i++) {
+			sb.append('\u00FC'); // ü
+		}
+		sb.append("\r\n");
+
+		String testString = sb.toString();
+		for (int i = 0; i < repetitions; i++) {
+			System.out.print(testString);
+		}
+	}
+}
diff --git a/org.eclipse.jdt.debug.tests/tests/org/eclipse/jdt/debug/tests/AbstractDebugTest.java b/org.eclipse.jdt.debug.tests/tests/org/eclipse/jdt/debug/tests/AbstractDebugTest.java
index 57e9c4c8a..a5444e184 100644
--- a/org.eclipse.jdt.debug.tests/tests/org/eclipse/jdt/debug/tests/AbstractDebugTest.java
+++ b/org.eclipse.jdt.debug.tests/tests/org/eclipse/jdt/debug/tests/AbstractDebugTest.java
@@ -207,7 +207,7 @@ public abstract class AbstractDebugTest extends TestCase implements  IEvaluation
 			"org.eclipse.debug.tests.targets.HcrClass5", "org.eclipse.debug.tests.targets.HcrClass6", "org.eclipse.debug.tests.targets.HcrClass7", "org.eclipse.debug.tests.targets.HcrClass8",
 			"org.eclipse.debug.tests.targets.HcrClass9", "TestContributedStepFilterClass", "TerminateAll_01", "TerminateAll_02", "StepResult1",
 			"StepResult2", "StepResult3", "StepUncaught", "TriggerPoint_01", "BulkThreadCreationTest", "MethodExitAndException",
-			"Bug534319earlyStart", "Bug534319lateStart", "Bug534319singleThread", "Bug534319startBetwen", "MethodCall", "Bug538303", "Bug540243", "OutSync", "OutSync2" };
+			"Bug534319earlyStart", "Bug534319lateStart", "Bug534319singleThread", "Bug534319startBetwen", "MethodCall", "Bug538303", "Bug540243", "OutSync", "OutSync2", "ConsoleOutputUmlaut" };
 
 	/**
 	 * the default timeout
diff --git a/org.eclipse.jdt.debug.tests/tests/org/eclipse/jdt/debug/tests/core/ConsoleInputTests.java b/org.eclipse.jdt.debug.tests/tests/org/eclipse/jdt/debug/tests/core/ConsoleInputTests.java
index 3a3f5df56..37377bc2e 100644
--- a/org.eclipse.jdt.debug.tests/tests/org/eclipse/jdt/debug/tests/core/ConsoleInputTests.java
+++ b/org.eclipse.jdt.debug.tests/tests/org/eclipse/jdt/debug/tests/core/ConsoleInputTests.java
@@ -1,5 +1,5 @@
 /*******************************************************************************
- * Copyright (c) 2000, 2015 IBM Corporation and others.
+ * Copyright (c) 2000, 2019 IBM Corporation and others.
  *
  * This program and the accompanying materials
  * are made available under the terms of the Eclipse Public License 2.0
@@ -13,12 +13,16 @@
  *******************************************************************************/
 package org.eclipse.jdt.debug.tests.core;
 
+import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
 
 import org.eclipse.core.runtime.Platform;
+import org.eclipse.debug.core.DebugPlugin;
 import org.eclipse.debug.core.ILaunch;
 import org.eclipse.debug.core.ILaunchConfiguration;
+import org.eclipse.debug.core.ILaunchConfigurationWorkingCopy;
 import org.eclipse.debug.core.ILaunchManager;
 import org.eclipse.debug.core.model.IStreamsProxy;
 import org.eclipse.debug.core.model.IStreamsProxy2;
@@ -57,7 +61,9 @@ public class ConsoleInputTests extends AbstractDebugTest implements IConsoleLine
 		return new OrderedTestSuite(ConsoleInputTests.class, new String[] {
 				"testMultiLineInput",
 				"testEOF",
-				"testDeleteAllEnteredText"
+				"testDeleteAllEnteredText",
+				"testBug545769_UTF8InEven",
+				"testBug545769_UTF8InOdd",
 		});
 	}
 
@@ -323,6 +329,68 @@ public class ConsoleInputTests extends AbstractDebugTest implements IConsoleLine
 		}
 	}
 
+	/**
+	 * Test if two byte UTF-8 characters get disrupted on there way to the running process input.
+	 * <p>
+	 * This test starts every two byte character on an even byte offset.
+	 * </p>
+	 *
+	 * @throws Exception
+	 *             if the test gets in trouble
+	 */
+	public void testBug545769_UTF8InEven() throws Exception {
+		// 4200 characters result in 8400 bytes which should be more than most common buffer sizes.
+		utf8InputTest("", 4200);
+	}
+
+	/**
+	 * Test if two byte UTF-8 characters get disrupted on there way to the running process input.
+	 * <p>
+	 * This test starts every two byte character on an odd byte offset.
+	 * </p>
+	 *
+	 * @throws Exception
+	 *             if the test gets in trouble
+	 */
+	public void testBug545769_UTF8InOdd() throws Exception {
+		// 4200 characters result in 8400 bytes which should be more than most common buffer sizes.
+		utf8InputTest(">", 4200);
+	}
+
+	/**
+	 * Shared code for the UTF-8 input tests.
+	 * <p>
+	 * Send some two byte UTF-8 characters to process and read the echo back.
+	 * </p>
+	 *
+	 * @param prefix
+	 *            an arbitrary prefix inserted before the two byte UTF-8 characters. Used to move the other characters to specific offsets e.g. a
+	 *            prefix of one byte will produce an input string where every two byte character starts at an odd offset.
+	 * @param numTwoByteCharacters
+	 *            number of two byte UTF-8 characters to send to process
+	 * @throws Exception
+	 *             if the test gets in trouble
+	 */
+	private void utf8InputTest(String prefix, int numTwoByteCharacters) throws Exception {
+		ConsoleLineTracker.setDelegate(this);
+		ILaunchConfiguration configuration = getLaunchConfiguration("ConsoleInput");
+		ILaunchConfigurationWorkingCopy configurationCopy = configuration.getWorkingCopy();
+		configurationCopy.setAttribute(DebugPlugin.ATTR_CONSOLE_ENCODING, StandardCharsets.UTF_8.name());
+		ILaunch launch = null;
+		try {
+			launch = configurationCopy.launch(ILaunchManager.RUN_MODE, null);
+			String input = prefix + String.join("", Collections.nCopies(numTwoByteCharacters, "\u00F8"));
+			waitStarted();
+			String[] list = appendAndGet(fConsole, input + "\n", 2);
+			verifyOutput(new String[] { input, input }, list);
+
+		} finally {
+			ConsoleLineTracker.setDelegate(null);
+			launch.getProcesses()[0].terminate();
+			getLaunchManager().removeLaunch(launch);
+		}
+	}
+
 	private void spinEventLoop() {
 		final Display display= DebugUIPlugin.getStandardDisplay();
 		Runnable runnable= new Runnable() {
diff --git a/org.eclipse.jdt.debug.tests/tests/org/eclipse/jdt/debug/tests/core/ConsoleTests.java b/org.eclipse.jdt.debug.tests/tests/org/eclipse/jdt/debug/tests/core/ConsoleTests.java
index 7f5b3ac22..39eb1e0d2 100644
--- a/org.eclipse.jdt.debug.tests/tests/org/eclipse/jdt/debug/tests/core/ConsoleTests.java
+++ b/org.eclipse.jdt.debug.tests/tests/org/eclipse/jdt/debug/tests/core/ConsoleTests.java
@@ -13,6 +13,7 @@
  *******************************************************************************/
 package org.eclipse.jdt.debug.tests.core;
 
+import java.nio.charset.StandardCharsets;
 import java.util.Arrays;
 import java.util.Collections;
 
@@ -23,10 +24,13 @@ import org.eclipse.debug.core.ILaunchConfigurationWorkingCopy;
 import org.eclipse.debug.core.ILaunchManager;
 import org.eclipse.debug.core.model.IProcess;
 import org.eclipse.debug.internal.ui.DebugUIPlugin;
+import org.eclipse.debug.internal.ui.preferences.IDebugPreferenceConstants;
 import org.eclipse.debug.ui.DebugUITools;
 import org.eclipse.jdt.debug.core.IJavaDebugTarget;
 import org.eclipse.jdt.debug.tests.AbstractDebugTest;
 import org.eclipse.jdt.debug.tests.TestUtil;
+import org.eclipse.jdt.launching.IJavaLaunchConfigurationConstants;
+import org.eclipse.jface.preference.IPreferenceStore;
 import org.eclipse.jface.text.IDocument;
 import org.eclipse.jface.text.IDocumentPartitioner;
 import org.eclipse.ui.console.ConsolePlugin;
@@ -37,7 +41,7 @@ import org.eclipse.ui.console.TextConsole;
 import org.eclipse.ui.internal.console.IOConsolePartitioner;
 
 /**
- * Tests console line tracker.
+ * Tests console lifecycle and output handling.
  */
 public class ConsoleTests extends AbstractDebugTest {
 
@@ -231,4 +235,71 @@ public class ConsoleTests extends AbstractDebugTest {
 		final IDocument consoleDocument = textConsole.getDocument();
 		return consoleDocument.get();
 	}
+	/**
+	 * Test console receiving UTF-8 output from process where two-byte UTF-8 characters start at even offsets.
+	 *
+	 * @throws Exception
+	 *             if the test gets in trouble
+	 */
+	public void testBug545769_UTF8OutEven() throws Exception {
+		// 4200 umlaute results in 8400 byte of output which should be more than most common buffer sizes.
+		utf8OutputTest(0, 4200, 5);
+	}
+
+	/**
+	 * Test console receiving UTF-8 output from process where two-byte UTF-8 characters start at odd offsets.
+	 *
+	 * @throws Exception
+	 *             if the test gets in trouble
+	 */
+	public void testBug545769_UTF8OutOdd() throws Exception {
+		// 4200 umlaute results in 8400 byte of output which should be more than most common buffer sizes.
+		utf8OutputTest(1, 4200, 5);
+	}
+
+	/**
+	 * Shared test code for possible UTF-8 process output corruption.
+	 *
+	 * @param numAscii
+	 *            number of one byte UTF-8 characters the process prints first
+	 * @param numUmlaut
+	 *            number of two byte UTF-8 character the process prints second
+	 * @param repetitions
+	 *            number of output repetitions. This test requires the process can write its output faster than the console can read it.
+	 * @throws Exception
+	 *             if the test gets in trouble
+	 */
+	private void utf8OutputTest(int numAscii, int numUmlaut, int repetitions) throws Exception {
+		final String typeName = "ConsoleOutputUmlaut";
+
+		final IPreferenceStore debugPrefStore = DebugUIPlugin.getDefault().getPreferenceStore();
+		debugPrefStore.setValue(IDebugPreferenceConstants.CONSOLE_LIMIT_CONSOLE_OUTPUT, false);
+		debugPrefStore.setValue(IDebugPreferenceConstants.CONSOLE_WRAP, true);
+		debugPrefStore.setValue(IDebugPreferenceConstants.CONSOLE_WIDTH, 100);
+
+		final ILaunchConfiguration launchConfig = getLaunchConfiguration(typeName);
+		final ILaunchConfigurationWorkingCopy launchCopy = launchConfig.getWorkingCopy();
+		String arg = String.join(" ", Integer.toString(numAscii), Integer.toString(numUmlaut), Integer.toString(repetitions));
+		launchCopy.setAttribute(IJavaLaunchConfigurationConstants.ATTR_PROGRAM_ARGUMENTS, arg);
+		launchCopy.setAttribute(DebugPlugin.ATTR_CONSOLE_ENCODING, StandardCharsets.UTF_8.name());
+
+		IJavaDebugTarget target = null;
+		try {
+			target = launchAndTerminate(launchCopy.doSave(), DEFAULT_TIMEOUT);
+			final IProcess process = target.getProcess();
+			assertNotNull("Missing VM process", process);
+			final IConsole console = DebugUITools.getConsole(process);
+			assertNotNull("Missing console", console);
+			assertTrue("Console is not a TextConsole", console instanceof TextConsole);
+			final TextConsole textConsole = (TextConsole) console;
+			TestUtil.waitForJobs(getName(), 100, DEFAULT_TIMEOUT); // wait for output appending
+			assertEquals("Test program failed with error.", 0, process.getExitValue());
+			final IDocument consoleDocument = textConsole.getDocument();
+			assertEquals("Wrong number of characters in console.", (numAscii + numUmlaut + 2) * repetitions, consoleDocument.getLength());
+		} finally {
+			terminateAndRemove(target);
+			debugPrefStore.setValue(IDebugPreferenceConstants.CONSOLE_LIMIT_CONSOLE_OUTPUT, true);
+			debugPrefStore.setValue(IDebugPreferenceConstants.CONSOLE_WRAP, false);
+		}
+	}
 }