Bug 507664 - IOConsoleOutputStream does not handle multi-byte charactersI20170301-2000

at buffer boundaries correctly

Change-Id: Ib1651069ab6a1a09d26e0b33bfae2dc3aef2fd77
Signed-off-by: Andreas Loth <andy_2639@justmail.de>

5 files changed, 243 insertions, 47 deletions

diff --git a/org.eclipse.ui.console/src/org/eclipse/ui/console/IOConsole.java b/org.eclipse.ui.console/src/org/eclipse/ui/console/IOConsole.java
index 1f573f70f..44cbfc4d7 100644
--- a/org.eclipse.ui.console/src/org/eclipse/ui/console/IOConsole.java
+++ b/org.eclipse.ui.console/src/org/eclipse/ui/console/IOConsole.java
@@ -1,5 +1,5 @@
 /*******************************************************************************
- * Copyright (c) 2000, 2014 IBM Corporation and others.
+ * Copyright (c) 2000, 2017 IBM Corporation and others.
  * All rights reserved. This program and the accompanying materials
  * are made available under the terms of the Eclipse Public License v1.0
  * which accompanies this distribution, and is available at
@@ -15,12 +15,12 @@ package org.eclipse.ui.console;
 import java.io.Closeable;
 import java.io.IOException;
 import java.io.InputStream;
+import java.nio.charset.Charset;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 
 import org.eclipse.jface.resource.ImageDescriptor;
-
 import org.eclipse.ui.WorkbenchEncoding;
 import org.eclipse.ui.internal.console.IOConsolePage;
 import org.eclipse.ui.internal.console.IOConsolePartitioner;
@@ -54,7 +54,7 @@ public class IOConsole extends TextConsole {
     /**
      * The encoding used to for displaying console output.
      */
-    private String fEncoding = WorkbenchEncoding.getWorkbenchDefaultEncoding();
+	private Charset charset;
 
 
     /**
@@ -68,7 +68,7 @@ public class IOConsole extends TextConsole {
      *  when this console is added/removed from the console manager
      */
     public IOConsole(String name, String consoleType, ImageDescriptor imageDescriptor, boolean autoLifecycle) {
-        this(name, consoleType, imageDescriptor, null, autoLifecycle);
+		this(name, consoleType, imageDescriptor, (String)null, autoLifecycle);
     }
 
     /**
@@ -83,10 +83,31 @@ public class IOConsole extends TextConsole {
      *  when this console is added/removed from the console manager
      */
     public IOConsole(String name, String consoleType, ImageDescriptor imageDescriptor, String encoding, boolean autoLifecycle) {
+		this(name, consoleType, imageDescriptor,
+				encoding == null
+						? Charset.forName(WorkbenchEncoding.getWorkbenchDefaultEncoding())
+						: Charset.forName(encoding),
+				autoLifecycle);
+    }
+
+    /**
+	 * Constructs a console with the given name, type, image, encoding and
+	 * lifecycle.
+	 *
+	 * @param name name to display for this console
+	 * @param consoleType console type identifier or <code>null</code>
+	 * @param imageDescriptor image to display for this console or
+	 *            <code>null</code>
+	 * @param charset the encoding that should be used to render the text, must
+	 *            not be <code>null</code>
+	 * @param autoLifecycle whether lifecycle methods should be called
+	 *            automatically when this console is added/removed from the
+	 *            console manager
+	 * @since 3.7
+	 */
+	public IOConsole(String name, String consoleType, ImageDescriptor imageDescriptor, Charset charset, boolean autoLifecycle) {
         super(name, consoleType, imageDescriptor, autoLifecycle);
-        if (encoding != null) {
-            fEncoding = encoding;
-        }
+		this.charset = charset;
         synchronized (openStreams) {
 			inputStream = new IOConsoleInputStream(this);
         	openStreams.add(inputStream);
@@ -144,8 +165,7 @@ public class IOConsole extends TextConsole {
      * @return a new output stream connected to this console
      */
     public IOConsoleOutputStream newOutputStream() {
-        IOConsoleOutputStream outputStream = new IOConsoleOutputStream(this);
-        outputStream.setEncoding(fEncoding);
+		IOConsoleOutputStream outputStream = new IOConsoleOutputStream(this, this.charset);
         synchronized(openStreams) {
             openStreams.add(outputStream);
         }
@@ -301,6 +321,19 @@ public class IOConsole extends TextConsole {
      * @since 3.3
      */
 	public String getEncoding() {
-		return fEncoding;
+		return this.charset.name();
 	}
+
+	/**
+	 * Returns the Charset for this console, or <code>null</code> to indicate
+	 * default encoding.
+	 *
+	 * @return the Charset for this console, or <code>null</code> to indicate
+	 *         default encoding
+	 * @since 3.7
+	 */
+	public Charset getCharset() {
+		return this.charset;
+	}
+
 }
diff --git a/org.eclipse.ui.console/src/org/eclipse/ui/console/IOConsoleOutputStream.java b/org.eclipse.ui.console/src/org/eclipse/ui/console/IOConsoleOutputStream.java
index 08efd3bfd..7e78ddf6a 100644
--- a/org.eclipse.ui.console/src/org/eclipse/ui/console/IOConsoleOutputStream.java
+++ b/org.eclipse.ui.console/src/org/eclipse/ui/console/IOConsoleOutputStream.java
@@ -13,10 +13,11 @@ package org.eclipse.ui.console;
 
 import java.io.IOException;
 import java.io.OutputStream;
+import java.nio.charset.Charset;
 
 import org.eclipse.swt.graphics.Color;
-import org.eclipse.ui.WorkbenchEncoding;
 import org.eclipse.ui.internal.console.IOConsolePartitioner;
+import org.eclipse.ui.internal.console.StreamDecoder;
 
 /**
  * OutputStream used to write to an IOConsole.
@@ -66,10 +67,7 @@ public class IOConsoleOutputStream extends OutputStream {
      */
     private int fontStyle;
 
-    private String fEncoding;
-    private String fDefaultEncoding = WorkbenchEncoding.getWorkbenchDefaultEncoding();
-
-    private boolean fNeedsEncoding = false;
+	private StreamDecoder decoder;
 
     private boolean prependCR;
 
@@ -78,8 +76,9 @@ public class IOConsoleOutputStream extends OutputStream {
      *
      * @param console I/O console
      */
-    IOConsoleOutputStream(IOConsole console) {
-        this.console = console;
+	IOConsoleOutputStream(IOConsole console, Charset charset) {
+		this.decoder = new StreamDecoder(charset);
+		this.console = console;
         this.partitioner = (IOConsolePartitioner) console.getPartitioner();
     }
 
@@ -195,11 +194,9 @@ public class IOConsoleOutputStream extends OutputStream {
      */
     @Override
 	public void write(byte[] b, int off, int len) throws IOException {
-        if (fNeedsEncoding) {
-            encodedWrite(new String(b, off, len, fEncoding));
-        } else {
-            encodedWrite(new String(b, off, len));
-        }
+		StringBuilder builder = new StringBuilder();
+		this.decoder.decode(builder, b, off, len);
+		encodedWrite(builder.toString());
     }
     /*
      *  (non-Javadoc)
@@ -218,12 +215,51 @@ public class IOConsoleOutputStream extends OutputStream {
         write(new byte[] {(byte)b}, 0, 1);
     }
 
+	/**
+	 * Writes a character array to the attached console.
+	 *
+	 * @param buffer the char array to write to the attached console
+	 * @throws IOException if the stream is closed
+	 * @since 3.7
+	 */
+	public void write(char[] buffer) throws IOException {
+		String str = new String(buffer);
+		this.encodedWrite(str);
+	}
+
+	/**
+	 * Writes a character array using specified offset and length to the
+	 * attached console.
+	 *
+	 * @param buffer the char array to write to the attached console.
+	 * @param off the initial offset
+	 * @param len the length
+	 * @throws IOException if the stream is closed
+	 * @since 3.7
+	 */
+	public void write(char[] buffer, int off, int len) throws IOException {
+		String str = new String(buffer, off, len);
+		this.encodedWrite(str);
+	}
+
     /**
-     * Writes a string to the attached console.
-     *
-     * @param str the string to write to the attached console.
-     * @throws IOException if the stream is closed.
-     */
+	 * Writes a character sequence to the attached console.
+	 *
+	 * @param chars the string/characters to write to the attached console.
+	 * @throws IOException if the stream is closed.
+	 * @since 3.7
+	 */
+	public void write(CharSequence chars) throws IOException {
+		String str = chars.toString();
+		encodedWrite(str);
+    }
+
+    /**
+	 * Writes a string to the attached console.
+	 *
+	 * @param str the string to write to the attached console
+	 * @throws IOException if the stream is closed
+	 */
     public void write(String str) throws IOException {
         encodedWrite(str);
     }
@@ -267,7 +303,25 @@ public class IOConsoleOutputStream extends OutputStream {
      * @param encoding encoding identifier
      */
     public void setEncoding(String encoding) {
-        fEncoding = encoding;
-        fNeedsEncoding = (fEncoding!=null) && (!fEncoding.equals(fDefaultEncoding));
+		Charset charset = Charset.forName(encoding);
+		try {
+			this.setCharset(charset);
+		} catch (IOException ioe) {
+			// ignore exception while writing final characters
+			// to avoid API break
+		}
+	}
+
+	/**
+	 * @param charset set the Charset for the attached console
+	 * @throws IOException if the stream is closed
+	 * @since 3.7
+	 */
+	public void setCharset(Charset charset) throws IOException {
+		StringBuilder builder = new StringBuilder();
+		this.decoder.finish(builder);
+		this.encodedWrite(builder.toString());
+		this.decoder = new StreamDecoder(charset);
     }
+
 }
diff --git a/org.eclipse.ui.console/src/org/eclipse/ui/console/MessageConsole.java b/org.eclipse.ui.console/src/org/eclipse/ui/console/MessageConsole.java
index d7ea0d0f2..aa399164e 100644
--- a/org.eclipse.ui.console/src/org/eclipse/ui/console/MessageConsole.java
+++ b/org.eclipse.ui.console/src/org/eclipse/ui/console/MessageConsole.java
@@ -1,5 +1,5 @@
 /*******************************************************************************
- * Copyright (c) 2000, 2013 IBM Corporation and others.
+ * Copyright (c) 2000, 2017 IBM Corporation and others.
  * All rights reserved. This program and the accompanying materials
  * are made available under the terms of the Eclipse Public License v1.0
  * which accompanies this distribution, and is available at
@@ -124,7 +124,7 @@ public class MessageConsole extends IOConsole {
 	 * @return a new message stream connected to this console
 	 */
 	public MessageConsoleStream newMessageStream() {
-		return new MessageConsoleStream(this);
+		return new MessageConsoleStream(this, this.getCharset());
 	}
 
     /* (non-Javadoc)
diff --git a/org.eclipse.ui.console/src/org/eclipse/ui/console/MessageConsoleStream.java b/org.eclipse.ui.console/src/org/eclipse/ui/console/MessageConsoleStream.java
index 274e8a8f2..c8593d53a 100644
--- a/org.eclipse.ui.console/src/org/eclipse/ui/console/MessageConsoleStream.java
+++ b/org.eclipse.ui.console/src/org/eclipse/ui/console/MessageConsoleStream.java
@@ -1,5 +1,5 @@
 /*******************************************************************************
- * Copyright (c) 2000, 2008 IBM Corporation and others.
+ * Copyright (c) 2000, 20017 IBM Corporation and others.
  * All rights reserved. This program and the accompanying materials
  * are made available under the terms of the Eclipse Public License v1.0
  * which accompanies this distribution, and is available at
@@ -11,6 +11,9 @@
 package org.eclipse.ui.console;
 
 import java.io.IOException;
+import java.nio.charset.Charset;
+
+import org.eclipse.ui.WorkbenchEncoding;
 
 /**
  * Used to write messages to a message console. A message console may have more
@@ -33,22 +36,32 @@ import java.io.IOException;
  * @noextend This class is not intended to be subclassed by clients.
  */
 public class MessageConsoleStream extends IOConsoleOutputStream {
-    
+
     private MessageConsole fMessageConsole;
-    
+
 	/**
 	 * Constructs a new stream connected to the given console.
-	 * 
+	 *
 	 * @param console the console to write messages to
 	 */
 	public MessageConsoleStream(MessageConsole console) {
-	    super(console);
+		this(console, Charset.forName(WorkbenchEncoding.getWorkbenchDefaultEncoding()));
+	}
+
+	/**
+	 * Constructs a new stream connected to the given console.
+	 *
+	 * @param console the console to write messages to
+	 * @since 3.7
+	 */
+	public MessageConsoleStream(MessageConsole console, Charset charset) {
+		super(console, charset);
         fMessageConsole = console;
 	}
-	
+
 	/**
 	 * Appends the specified message to this stream.
-	 * 
+	 *
 	 * @param message message to append
 	 */
 	public void print(String message) {
@@ -58,8 +71,8 @@ public class MessageConsoleStream extends IOConsoleOutputStream {
             ConsolePlugin.log(e);
         }
 	}
-	
-	
+
+
 	/**
 	 * Appends a line separator string to this stream.
 	 */
@@ -69,24 +82,24 @@ public class MessageConsoleStream extends IOConsoleOutputStream {
         } catch (IOException e) {
             ConsolePlugin.log(e);
         }
-	}	
-	
+	}
+
 	/**
 	 * Appends the specified message to this stream, followed by a line
 	 * separator string.
-	 * 
+	 *
 	 * @param message message to print
 	 */
 	public void println(String message) {
 		print(message + "\n"); //$NON-NLS-1$
-	}	
-    
+	}
+
     /**
      * Returns the console this stream is connected to.
-     * 
+     *
      * @return the console this stream is connected to
      */
     public MessageConsole getConsole() {
         return fMessageConsole;
-    }    
+    }
 }
diff --git a/org.eclipse.ui.console/src/org/eclipse/ui/internal/console/StreamDecoder.java b/org.eclipse.ui.console/src/org/eclipse/ui/internal/console/StreamDecoder.java
new file mode 100644
index 000000000..dde14fdcd
--- /dev/null
+++ b/org.eclipse.ui.console/src/org/eclipse/ui/internal/console/StreamDecoder.java
@@ -0,0 +1,96 @@
+/*******************************************************************************
+ * Copyright (c) 2017 Andreas Loth and others.
+ * All rights reserved. This program and the accompanying materials
+ * are made available under the terms of the Eclipse Public License v1.0
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/epl-v10.html
+ *
+ * Contributors:
+ *     Andreas Loth - initial API and implementation
+ *******************************************************************************/
+
+package org.eclipse.ui.internal.console;
+
+import java.nio.ByteBuffer;
+import java.nio.CharBuffer;
+import java.nio.charset.Charset;
+import java.nio.charset.CharsetDecoder;
+import java.nio.charset.CoderResult;
+import java.nio.charset.CodingErrorAction;
+
+
+/**
+ * @since 3.7
+ */
+public class StreamDecoder {
+
+	static private final int BUFFER_SIZE = 4096;
+
+	private final CharsetDecoder decoder;
+	private final ByteBuffer inputBuffer;
+	private final CharBuffer outputBuffer;
+
+	public StreamDecoder(Charset charset) {
+		this.decoder = charset.newDecoder();
+		this.decoder.onMalformedInput(CodingErrorAction.REPLACE);
+		this.decoder.onUnmappableCharacter(CodingErrorAction.REPLACE);
+		this.inputBuffer = ByteBuffer.allocate(StreamDecoder.BUFFER_SIZE);
+		this.inputBuffer.flip();
+		this.outputBuffer = CharBuffer.allocate(StreamDecoder.BUFFER_SIZE);
+	}
+
+	private void consume(StringBuilder consumer) {
+		this.outputBuffer.flip();
+		consumer.append(this.outputBuffer);
+		this.outputBuffer.clear();
+	}
+
+	private void internalDecode(StringBuilder consumer, byte[] buffer, int offset, int length, boolean last) {
+		assert (offset >= 0);
+		assert (length >= 0);
+		int position = offset;
+		int end = offset + length;
+		assert (end <= buffer.length);
+		boolean finished = false;
+		do {
+			CoderResult result = this.decoder.decode(this.inputBuffer, this.outputBuffer, last);
+			if (result.isOverflow()) {
+				this.consume(consumer);
+			} else if (result.isUnderflow()) {
+				this.inputBuffer.compact();
+				int remaining = this.inputBuffer.remaining();
+				assert (remaining > 0);
+				int read = Math.min(remaining, end - position);
+				if (read > 0) {
+					this.inputBuffer.put(buffer, position, read);
+					position += read;
+				} else {
+					finished = true;
+				}
+				this.inputBuffer.flip();
+			} else {
+				assert false;
+			}
+		} while (!finished);
+	}
+
+	public void decode(StringBuilder consumer, byte[] buffer, int offset, int length) {
+		this.internalDecode(consumer, buffer, offset, length, false);
+		this.consume(consumer);
+	}
+
+	public void finish(StringBuilder consumer) {
+		this.internalDecode(consumer, new byte[0], 0, 0, true);
+		CoderResult result;
+		do {
+			result = this.decoder.flush(this.outputBuffer);
+			if (result.isOverflow()) {
+				this.consume(consumer);
+			} else {
+				assert result.isUnderflow();
+			}
+		} while (!result.isUnderflow());
+		this.consume(consumer);
+	}
+
+}