Bug 485300 - Bundle manifest with double byte UTF chars are parsed

incorrectly with line continuations

Change-Id: I5f3c9a13db35683aa98820f990e8614acbc8b2ca
Signed-off-by: Thomas Watson <tjwatson@us.ibm.com>

3 files changed, 142 insertions, 37 deletions

diff --git a/bundles/org.eclipse.osgi.tests/src/org/eclipse/osgi/tests/container/TestModuleContainer.java b/bundles/org.eclipse.osgi.tests/src/org/eclipse/osgi/tests/container/TestModuleContainer.java
index a45d0825d..c13f2f9e6 100644
--- a/bundles/org.eclipse.osgi.tests/src/org/eclipse/osgi/tests/container/TestModuleContainer.java
+++ b/bundles/org.eclipse.osgi.tests/src/org/eclipse/osgi/tests/container/TestModuleContainer.java
@@ -10,9 +10,13 @@
  *******************************************************************************/
 package org.eclipse.osgi.tests.container;
 
+import static java.util.jar.Attributes.Name.MANIFEST_VERSION;
+
 import java.io.*;
 import java.util.*;
 import java.util.concurrent.*;
+import java.util.jar.Attributes;
+import java.util.jar.Manifest;
 import org.eclipse.osgi.container.*;
 import org.eclipse.osgi.container.Module.StartOptions;
 import org.eclipse.osgi.container.Module.State;
@@ -26,6 +30,7 @@ import org.eclipse.osgi.report.resolution.ResolutionReport;
 import org.eclipse.osgi.tests.container.dummys.*;
 import org.eclipse.osgi.tests.container.dummys.DummyModuleDatabase.DummyContainerEvent;
 import org.eclipse.osgi.tests.container.dummys.DummyModuleDatabase.DummyModuleEvent;
+import org.eclipse.osgi.util.ManifestElement;
 import org.junit.Assert;
 import org.junit.Test;
 import org.osgi.framework.*;
@@ -2076,6 +2081,44 @@ public class TestModuleContainer extends AbstractTest {
 	}
 
 	@Test
+	public void testUTF8LineContinuation() throws BundleException, IOException {
+		DummyContainerAdaptor adaptor = createDummyAdaptor();
+		ModuleContainer container = adaptor.getContainer();
+		String utfString = "a.with.é.multibyte";
+		while (utfString.getBytes("UTF8").length < 500) {
+			Map<String, String> manifest = getUTFManifest(utfString);
+			Module testModule = installDummyModule(manifest, manifest.get(Constants.BUNDLE_SYMBOLICNAME), container);
+			Assert.assertEquals("Wrong bns for the bundle.", utfString, testModule.getCurrentRevision().getSymbolicName());
+
+			ModuleCapability exportPackage = testModule.getCurrentRevision().getModuleCapabilities(PackageNamespace.PACKAGE_NAMESPACE).get(0);
+			ModuleRequirement importPackage = testModule.getCurrentRevision().getModuleRequirements(PackageNamespace.PACKAGE_NAMESPACE).get(0);
+
+			String actualPackageName = (String) exportPackage.getAttributes().get(PackageNamespace.PACKAGE_NAMESPACE);
+			Assert.assertEquals("Wrong exported package name.", utfString, actualPackageName);
+
+			Assert.assertTrue("import does not match export: " + importPackage, importPackage.matches(exportPackage));
+
+			utfString = "a" + utfString;
+		}
+	}
+
+	private static Map<String, String> getUTFManifest(String packageName) throws IOException, BundleException {
+		// using manifest class to force a split line right in the middle of a double byte UTF-8 character
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		{
+			Manifest m = new Manifest();
+			Attributes a = m.getMainAttributes();
+			a.put(MANIFEST_VERSION, "1.0");
+			a.putValue(Constants.BUNDLE_MANIFESTVERSION, "2");
+			a.putValue(Constants.BUNDLE_SYMBOLICNAME, packageName);
+			a.putValue(Constants.EXPORT_PACKAGE, packageName);
+			a.putValue(Constants.IMPORT_PACKAGE, packageName);
+			m.write(out);
+		}
+		return ManifestElement.parseBundleManifest(new ByteArrayInputStream(out.toByteArray()), null);
+	}
+
+	@Test
 	public void testBug483849() throws BundleException, IOException {
 		DummyContainerAdaptor adaptor = createDummyAdaptor();
 		ModuleContainer container = adaptor.getContainer();
diff --git a/bundles/org.eclipse.osgi.tests/src/org/eclipse/osgi/tests/util/ManifestElementTestCase.java b/bundles/org.eclipse.osgi.tests/src/org/eclipse/osgi/tests/util/ManifestElementTestCase.java
index dceb8b9a9..559673d70 100644
--- a/bundles/org.eclipse.osgi.tests/src/org/eclipse/osgi/tests/util/ManifestElementTestCase.java
+++ b/bundles/org.eclipse.osgi.tests/src/org/eclipse/osgi/tests/util/ManifestElementTestCase.java
@@ -1,5 +1,5 @@
 /*******************************************************************************
- * Copyright (c) 2006, 2008 IBM Corporation and others.
+ * Copyright (c) 2006, 2016 IBM Corporation and others.
  * All rights reserved. This program and the accompanying materials
  * are made available under the terms of the Eclipse Public License v1.0
  * which accompanies this distribution, and is available at
@@ -11,9 +11,13 @@
 
 package org.eclipse.osgi.tests.util;
 
+import java.io.*;
+import java.util.*;
 import junit.framework.TestCase;
 import org.eclipse.osgi.util.ManifestElement;
+import org.junit.Assert;
 import org.osgi.framework.BundleException;
+import org.osgi.framework.Constants;
 
 public class ManifestElementTestCase extends TestCase {
 
@@ -53,4 +57,40 @@ public class ManifestElementTestCase extends TestCase {
 		assertEquals("2.0", components[0], "external:test1:test2");
 		assertEquals("2.1", components[1], "test3:test4:");
 	}
+
+	private static final List<String> TEST_MANIFEST = Arrays.asList(//
+			"Bundle-ManifestVersion: 2", //
+			"Bundle-SymbolicName: test.", //
+			" bsn", //
+			"Import-Package: test1,", //
+			" test2,", //
+			" test3", //
+			"" //
+	);
+
+	public void testManifestWithCR() throws UnsupportedEncodingException, IOException, BundleException {
+		doManifestTest("\r");
+	}
+
+	public void testManifestWithLF() throws UnsupportedEncodingException, IOException, BundleException {
+		doManifestTest("\n");
+	}
+
+	public void testManifestWithCRLF() throws UnsupportedEncodingException, IOException, BundleException {
+		doManifestTest("\r\n");
+	}
+
+	private void doManifestTest(String newLine) throws UnsupportedEncodingException, IOException, BundleException {
+		Map<String, String> manifest = getManifest(TEST_MANIFEST, newLine);
+		Assert.assertEquals("Wrong Bundle-SymbolicName.", "test.bsn", manifest.get(Constants.BUNDLE_SYMBOLICNAME));
+		Assert.assertEquals("Wrong Import-Package.", "test1,test2,test3", manifest.get(Constants.IMPORT_PACKAGE));
+	}
+
+	private Map<String, String> getManifest(List<String> manifestLines, String newLine) throws UnsupportedEncodingException, IOException, BundleException {
+		StringBuilder manifestText = new StringBuilder();
+		for (String line : manifestLines) {
+			manifestText.append(line).append(newLine);
+		}
+		return ManifestElement.parseBundleManifest(new ByteArrayInputStream(manifestText.toString().getBytes("UTF8")), null);
+	}
 }
diff --git a/bundles/org.eclipse.osgi/supplement/src/org/eclipse/osgi/util/ManifestElement.java b/bundles/org.eclipse.osgi/supplement/src/org/eclipse/osgi/util/ManifestElement.java
index 1ef321071..10616c3a9 100644
--- a/bundles/org.eclipse.osgi/supplement/src/org/eclipse/osgi/util/ManifestElement.java
+++ b/bundles/org.eclipse.osgi/supplement/src/org/eclipse/osgi/util/ManifestElement.java
@@ -1,5 +1,5 @@
 /*******************************************************************************
- * Copyright (c) 2003, 2013 IBM Corporation and others.
+ * Copyright (c) 2003, 2016 IBM Corporation and others.
  * All rights reserved. This program and the accompanying materials
  * are made available under the terms of the Eclipse Public License v1.0
  * which accompanies this distribution, and is available at
@@ -376,7 +376,7 @@ public class ManifestElement {
 					} else
 						directive = true;
 				}
-				if (c == ';' || c == ',' || c == '\0') /* more */{
+				if (c == ';' || c == ',' || c == '\0') /* more */ {
 					headerValues.add(next);
 					headerValue.append(";").append(next); //$NON-NLS-1$
 					if (SupplementDebug.STATIC_DEBUG_MANIFEST)
@@ -425,7 +425,7 @@ public class ManifestElement {
 					throw new BundleException(NLS.bind(Msg.MANIFEST_INVALID_HEADER_EXCEPTION, header, value), BundleException.MANIFEST_ERROR, e);
 				}
 				c = tokenizer.getChar();
-				if (c == ';') /* more */{
+				if (c == ';') /* more */ {
 					next = tokenizer.getToken("=:"); //$NON-NLS-1$
 					if (next == null)
 						throw new BundleException(NLS.bind(Msg.MANIFEST_INVALID_HEADER_EXCEPTION, header, value), BundleException.MANIFEST_ERROR);
@@ -501,19 +501,13 @@ public class ManifestElement {
 	public static Map<String, String> parseBundleManifest(InputStream manifest, Map<String, String> headers) throws IOException, BundleException {
 		if (headers == null)
 			headers = new HashMap<String, String>();
-		BufferedReader br;
-		try {
-			br = new BufferedReader(new InputStreamReader(manifest, "UTF8")); //$NON-NLS-1$
-		} catch (UnsupportedEncodingException e) {
-			br = new BufferedReader(new InputStreamReader(manifest));
-		}
+
+		manifest = new BufferedInputStream(manifest);
 		try {
-			String header = null;
-			StringBuffer value = new StringBuffer(256);
-			boolean firstLine = true;
 
+			ByteArrayOutputStream buffer = new ByteArrayOutputStream(256);
 			while (true) {
-				String line = br.readLine();
+				String line = readLine(manifest, buffer);
 				/* The java.util.jar classes in JDK 1.3 use the value of the last
 				 * encountered manifest header. So we do the same to emulate
 				 * this behavior. We no longer throw a BundleException
@@ -522,40 +516,21 @@ public class ManifestElement {
 
 				if ((line == null) || (line.length() == 0)) /* EOF or empty line */
 				{
-					if (!firstLine) /* flush last line */
-					{
-						headers.put(header, value.toString().trim());
-					}
 					break; /* done processing main attributes */
 				}
 
-				if (line.charAt(0) == ' ') /* continuation */
-				{
-					if (firstLine) /* if no previous line */
-					{
-						throw new BundleException(NLS.bind(Msg.MANIFEST_INVALID_SPACE, line), BundleException.MANIFEST_ERROR);
-					}
-					value.append(line.substring(1));
-					continue;
-				}
-
-				if (!firstLine) {
-					headers.put(header, value.toString().trim());
-					value.setLength(0); /* clear StringBuffer */
-				}
-
 				int colon = line.indexOf(':');
 				if (colon == -1) /* no colon */
 				{
 					throw new BundleException(NLS.bind(Msg.MANIFEST_INVALID_LINE_NOCOLON, line), BundleException.MANIFEST_ERROR);
 				}
-				header = line.substring(0, colon).trim();
-				value.append(line.substring(colon + 1));
-				firstLine = false;
+				String header = line.substring(0, colon).trim();
+				String value = line.substring(colon + 1).trim();
+				headers.put(header, value);
 			}
 		} finally {
 			try {
-				br.close();
+				manifest.close();
 			} catch (IOException ee) {
 				// do nothing
 			}
@@ -563,6 +538,53 @@ public class ManifestElement {
 		return headers;
 	}
 
+	private static String readLine(InputStream input, ByteArrayOutputStream buffer) throws IOException {
+		// Read a header 'line'
+		// A header line may span multiple lines with line continuations using a beginning space.
+		// This method reads all the line continuations into a single string.
+		// Care must be taken for cases where double byte UTF characters are split
+		// across line continuations.
+		// This is why BufferedReader.readLine is not used here.  We must process the 
+		// CR LF chars ourselves
+		lineLoop: while (true) {
+			int c = input.read();
+			if (c == '\n') { // LF
+				// next char is either a continuation (space) char or the first char of the next header
+				input.mark(1);
+				c = input.read();
+				if (c != ' ') {
+					// This first char of the next header, reset so we don't loose the char
+					input.reset();
+					break lineLoop;
+				}
+				// This is a continuation, skip the space and read the next char
+				c = input.read();
+			} else if (c == '\r') { // CR
+				// next char is either a continuation (space) char, LF or the first char of the next header
+				input.mark(1);
+				c = input.read();
+				if (c == '\n') { // LF
+					// next char is either a continuation (space) char or the first char of the next header
+					input.mark(1);
+					c = input.read();
+				}
+				if (c != ' ') {
+					// This first char of the next header, reset so we don't loose the char
+					input.reset();
+					break lineLoop;
+				}
+				c = input.read();
+			}
+			if (c == -1) {
+				break lineLoop;
+			}
+			buffer.write(c);
+		}
+		String result = buffer.toString("UTF8"); //$NON-NLS-1$
+		buffer.reset();
+		return result;
+	}
+
 	public String toString() {
 		Enumeration<String> attrKeys = getKeys();
 		Enumeration<String> directiveKeys = getDirectiveKeys();