bug 363380: textile fails when & and ; on the same line task-363380-entity-reference

https://bugs.eclipse.org/bugs/show_bug.cgi?id=363380

Change-Id: I0f6965afaec2db4b2debc83246be4342ad722a2c

3 files changed, 180 insertions, 2 deletions

diff --git a/org.eclipse.mylyn.wikitext.core/src/org/eclipse/mylyn/wikitext/core/parser/builder/HtmlDocumentBuilder.java b/org.eclipse.mylyn.wikitext.core/src/org/eclipse/mylyn/wikitext/core/parser/builder/HtmlDocumentBuilder.java
index 2bdd0eee9..698e56002 100644
--- a/org.eclipse.mylyn.wikitext.core/src/org/eclipse/mylyn/wikitext/core/parser/builder/HtmlDocumentBuilder.java
+++ b/org.eclipse.mylyn.wikitext.core/src/org/eclipse/mylyn/wikitext/core/parser/builder/HtmlDocumentBuilder.java
@@ -48,6 +48,107 @@ import org.eclipse.mylyn.wikitext.core.util.XmlStreamWriter;
 public class HtmlDocumentBuilder extends AbstractXmlDocumentBuilder {
 	private static final Pattern ABSOLUTE_URL_PATTERN = Pattern.compile("[a-zA-Z]{3,8}://?.*"); //$NON-NLS-1$
 
+	private static final Map<String, String> entityReferenceToNumericEquivalent = new HashMap<String, String>();
+	static {
+		entityReferenceToNumericEquivalent.put("nbsp", "#160"); //$NON-NLS-1$ //$NON-NLS-2$
+		entityReferenceToNumericEquivalent.put("iexcl", "#161");//$NON-NLS-1$ //$NON-NLS-2$
+		entityReferenceToNumericEquivalent.put("cent", "#162");//$NON-NLS-1$ //$NON-NLS-2$
+		entityReferenceToNumericEquivalent.put("pound", "#163");//$NON-NLS-1$ //$NON-NLS-2$
+		entityReferenceToNumericEquivalent.put("curren", "#164");//$NON-NLS-1$ //$NON-NLS-2$
+		entityReferenceToNumericEquivalent.put("yen", "#165");//$NON-NLS-1$ //$NON-NLS-2$
+		entityReferenceToNumericEquivalent.put("brvbar", "#166");//$NON-NLS-1$ //$NON-NLS-2$
+		entityReferenceToNumericEquivalent.put("sect", "#167");//$NON-NLS-1$ //$NON-NLS-2$
+		entityReferenceToNumericEquivalent.put("uml", "#168");//$NON-NLS-1$ //$NON-NLS-2$
+		entityReferenceToNumericEquivalent.put("copy", "#169");//$NON-NLS-1$ //$NON-NLS-2$
+		entityReferenceToNumericEquivalent.put("ordf", "#170");//$NON-NLS-1$ //$NON-NLS-2$
+		entityReferenceToNumericEquivalent.put("laquo", "#171");//$NON-NLS-1$ //$NON-NLS-2$
+		entityReferenceToNumericEquivalent.put("not", "#172");//$NON-NLS-1$ //$NON-NLS-2$
+		entityReferenceToNumericEquivalent.put("shy", "#173");//$NON-NLS-1$ //$NON-NLS-2$
+		entityReferenceToNumericEquivalent.put("reg", "#174");//$NON-NLS-1$ //$NON-NLS-2$
+		entityReferenceToNumericEquivalent.put("macr", "#175");//$NON-NLS-1$ //$NON-NLS-2$
+		entityReferenceToNumericEquivalent.put("deg", "#176");//$NON-NLS-1$ //$NON-NLS-2$
+		entityReferenceToNumericEquivalent.put("plusmn", "#177");//$NON-NLS-1$ //$NON-NLS-2$
+		entityReferenceToNumericEquivalent.put("sup2", "#178");//$NON-NLS-1$ //$NON-NLS-2$
+		entityReferenceToNumericEquivalent.put("sup3", "#179");//$NON-NLS-1$ //$NON-NLS-2$
+		entityReferenceToNumericEquivalent.put("acute", "#180");//$NON-NLS-1$ //$NON-NLS-2$
+		entityReferenceToNumericEquivalent.put("micro", "#181");//$NON-NLS-1$ //$NON-NLS-2$
+		entityReferenceToNumericEquivalent.put("para", "#182");//$NON-NLS-1$ //$NON-NLS-2$
+		entityReferenceToNumericEquivalent.put("middot", "#183");//$NON-NLS-1$ //$NON-NLS-2$
+		entityReferenceToNumericEquivalent.put("cedil", "#184");//$NON-NLS-1$ //$NON-NLS-2$
+		entityReferenceToNumericEquivalent.put("sup1", "#185");//$NON-NLS-1$ //$NON-NLS-2$
+		entityReferenceToNumericEquivalent.put("ordm", "#186");//$NON-NLS-1$ //$NON-NLS-2$
+		entityReferenceToNumericEquivalent.put("raquo", "#187");//$NON-NLS-1$ //$NON-NLS-2$
+		entityReferenceToNumericEquivalent.put("frac14", "#188");//$NON-NLS-1$ //$NON-NLS-2$
+		entityReferenceToNumericEquivalent.put("frac12", "#189");//$NON-NLS-1$ //$NON-NLS-2$
+		entityReferenceToNumericEquivalent.put("frac34", "#190");//$NON-NLS-1$ //$NON-NLS-2$
+		entityReferenceToNumericEquivalent.put("iquest", "#191");//$NON-NLS-1$ //$NON-NLS-2$
+		entityReferenceToNumericEquivalent.put("times", "#215");//$NON-NLS-1$ //$NON-NLS-2$
+		entityReferenceToNumericEquivalent.put("divide", "#247");//$NON-NLS-1$ //$NON-NLS-2$
+		entityReferenceToNumericEquivalent.put("Agrave", "#192");//$NON-NLS-1$ //$NON-NLS-2$
+		entityReferenceToNumericEquivalent.put("Aacute", "#193");//$NON-NLS-1$ //$NON-NLS-2$
+		entityReferenceToNumericEquivalent.put("Acirc", "#194");//$NON-NLS-1$ //$NON-NLS-2$
+		entityReferenceToNumericEquivalent.put("Atilde", "#195");//$NON-NLS-1$ //$NON-NLS-2$
+		entityReferenceToNumericEquivalent.put("Auml", "#196");//$NON-NLS-1$ //$NON-NLS-2$
+		entityReferenceToNumericEquivalent.put("Aring", "#197");//$NON-NLS-1$ //$NON-NLS-2$
+		entityReferenceToNumericEquivalent.put("AElig", "#198");//$NON-NLS-1$ //$NON-NLS-2$
+		entityReferenceToNumericEquivalent.put("Ccedil", "#199");//$NON-NLS-1$ //$NON-NLS-2$
+		entityReferenceToNumericEquivalent.put("Egrave", "#200");//$NON-NLS-1$ //$NON-NLS-2$
+		entityReferenceToNumericEquivalent.put("Eacute", "#201");//$NON-NLS-1$ //$NON-NLS-2$
+		entityReferenceToNumericEquivalent.put("Ecirc", "#202");//$NON-NLS-1$ //$NON-NLS-2$
+		entityReferenceToNumericEquivalent.put("Euml", "#203");//$NON-NLS-1$ //$NON-NLS-2$
+		entityReferenceToNumericEquivalent.put("Igrave", "#204");//$NON-NLS-1$ //$NON-NLS-2$
+		entityReferenceToNumericEquivalent.put("Iacute", "#205");//$NON-NLS-1$ //$NON-NLS-2$
+		entityReferenceToNumericEquivalent.put("Icirc", "#206");//$NON-NLS-1$ //$NON-NLS-2$
+		entityReferenceToNumericEquivalent.put("Iuml", "#207");//$NON-NLS-1$ //$NON-NLS-2$
+		entityReferenceToNumericEquivalent.put("ETH", "#208");//$NON-NLS-1$ //$NON-NLS-2$
+		entityReferenceToNumericEquivalent.put("Ntilde", "#209");//$NON-NLS-1$ //$NON-NLS-2$
+		entityReferenceToNumericEquivalent.put("Ograve", "#210");//$NON-NLS-1$ //$NON-NLS-2$
+		entityReferenceToNumericEquivalent.put("Oacute", "#211");//$NON-NLS-1$ //$NON-NLS-2$
+		entityReferenceToNumericEquivalent.put("Ocirc", "#212");//$NON-NLS-1$ //$NON-NLS-2$
+		entityReferenceToNumericEquivalent.put("Otilde", "#213");//$NON-NLS-1$ //$NON-NLS-2$
+		entityReferenceToNumericEquivalent.put("Ouml", "#214");//$NON-NLS-1$ //$NON-NLS-2$
+		entityReferenceToNumericEquivalent.put("Oslash", "#216");//$NON-NLS-1$ //$NON-NLS-2$
+		entityReferenceToNumericEquivalent.put("Ugrave", "#217");//$NON-NLS-1$ //$NON-NLS-2$
+		entityReferenceToNumericEquivalent.put("Uacute", "#218");//$NON-NLS-1$ //$NON-NLS-2$
+		entityReferenceToNumericEquivalent.put("Ucirc", "#219");//$NON-NLS-1$ //$NON-NLS-2$
+		entityReferenceToNumericEquivalent.put("Uuml", "#220");//$NON-NLS-1$ //$NON-NLS-2$
+		entityReferenceToNumericEquivalent.put("Yacute", "#221");//$NON-NLS-1$ //$NON-NLS-2$
+		entityReferenceToNumericEquivalent.put("THORN", "#222");//$NON-NLS-1$ //$NON-NLS-2$
+		entityReferenceToNumericEquivalent.put("szlig", "#223");//$NON-NLS-1$ //$NON-NLS-2$
+		entityReferenceToNumericEquivalent.put("agrave", "#224");//$NON-NLS-1$ //$NON-NLS-2$
+		entityReferenceToNumericEquivalent.put("aacute", "#225");//$NON-NLS-1$ //$NON-NLS-2$
+		entityReferenceToNumericEquivalent.put("acirc", "#226");//$NON-NLS-1$ //$NON-NLS-2$
+		entityReferenceToNumericEquivalent.put("atilde", "#227");//$NON-NLS-1$ //$NON-NLS-2$
+		entityReferenceToNumericEquivalent.put("auml", "#228");//$NON-NLS-1$ //$NON-NLS-2$
+		entityReferenceToNumericEquivalent.put("aring", "#229");//$NON-NLS-1$ //$NON-NLS-2$
+		entityReferenceToNumericEquivalent.put("aelig", "#230");//$NON-NLS-1$ //$NON-NLS-2$
+		entityReferenceToNumericEquivalent.put("ccedil", "#231");//$NON-NLS-1$ //$NON-NLS-2$
+		entityReferenceToNumericEquivalent.put("egrave", "#232");//$NON-NLS-1$ //$NON-NLS-2$
+		entityReferenceToNumericEquivalent.put("eacute", "#233");//$NON-NLS-1$ //$NON-NLS-2$
+		entityReferenceToNumericEquivalent.put("ecirc", "#234");//$NON-NLS-1$ //$NON-NLS-2$
+		entityReferenceToNumericEquivalent.put("euml", "#235");//$NON-NLS-1$ //$NON-NLS-2$
+		entityReferenceToNumericEquivalent.put("igrave", "#236");//$NON-NLS-1$ //$NON-NLS-2$
+		entityReferenceToNumericEquivalent.put("iacute", "#237");//$NON-NLS-1$ //$NON-NLS-2$
+		entityReferenceToNumericEquivalent.put("icirc", "#238");//$NON-NLS-1$ //$NON-NLS-2$
+		entityReferenceToNumericEquivalent.put("iuml", "#239");//$NON-NLS-1$ //$NON-NLS-2$
+		entityReferenceToNumericEquivalent.put("eth", "#240");//$NON-NLS-1$ //$NON-NLS-2$
+		entityReferenceToNumericEquivalent.put("ntilde", "#241");//$NON-NLS-1$ //$NON-NLS-2$
+		entityReferenceToNumericEquivalent.put("ograve", "#242");//$NON-NLS-1$ //$NON-NLS-2$
+		entityReferenceToNumericEquivalent.put("oacute", "#243");//$NON-NLS-1$ //$NON-NLS-2$
+		entityReferenceToNumericEquivalent.put("ocirc", "#244");//$NON-NLS-1$ //$NON-NLS-2$
+		entityReferenceToNumericEquivalent.put("otilde", "#245");//$NON-NLS-1$ //$NON-NLS-2$
+		entityReferenceToNumericEquivalent.put("ouml", "#246");//$NON-NLS-1$ //$NON-NLS-2$
+		entityReferenceToNumericEquivalent.put("oslash", "#248");//$NON-NLS-1$ //$NON-NLS-2$
+		entityReferenceToNumericEquivalent.put("ugrave", "#249");//$NON-NLS-1$ //$NON-NLS-2$
+		entityReferenceToNumericEquivalent.put("uacute", "#250");//$NON-NLS-1$ //$NON-NLS-2$
+		entityReferenceToNumericEquivalent.put("ucirc", "#251");//$NON-NLS-1$ //$NON-NLS-2$
+		entityReferenceToNumericEquivalent.put("uuml", "#252");//$NON-NLS-1$ //$NON-NLS-2$
+		entityReferenceToNumericEquivalent.put("yacute", "#253");//$NON-NLS-1$ //$NON-NLS-2$
+		entityReferenceToNumericEquivalent.put("thorn", "#254");//$NON-NLS-1$ //$NON-NLS-2$
+		entityReferenceToNumericEquivalent.put("yuml", "#255");//$NON-NLS-1$ //$NON-NLS-2$
+
+	}
+
 	private static final Map<SpanType, String> spanTypeToElementName = new HashMap<SpanType, String>();
 	static {
 		spanTypeToElementName.put(SpanType.LINK, "a"); //$NON-NLS-1$
@@ -124,6 +225,8 @@ public class HtmlDocumentBuilder extends AbstractXmlDocumentBuilder {
 
 	private String prependImagePrefix;
 
+	private boolean filterEntityReferences = false;
+
 	/**
 	 * construct the HtmlDocumentBuilder.
 	 * 
@@ -629,7 +732,20 @@ public class HtmlDocumentBuilder extends AbstractXmlDocumentBuilder {
 
 	@Override
 	public void entityReference(String entity) {
-		writer.writeEntityRef(entity);
+		if (filterEntityReferences) {
+			String emitEntity = entity.length() > 0 && entity.charAt(0) == '#'
+					? entity
+					: entityReferenceToNumericEquivalent.get(entity);
+			if (emitEntity == null) {
+				writer.writeCharacters("&"); //$NON-NLS-1$
+				writer.writeCharacters(entity);
+				writer.writeCharacters(";"); //$NON-NLS-1$
+			} else {
+				writer.writeEntityRef(emitEntity);
+			}
+		} else {
+			writer.writeEntityRef(entity);
+		}
 	}
 
 	@Override
@@ -1199,4 +1315,28 @@ public class HtmlDocumentBuilder extends AbstractXmlDocumentBuilder {
 		return prependImagePrefix;
 	}
 
+	/**
+	 * Indicates that {@link #entityReference(String) entity references} should be filtered. Defaults to false. When
+	 * filtered, known HTML entity references are converted to their numeric counterpart, and unknown entity references
+	 * are emitted as plain text.
+	 * 
+	 * @since 1.6
+	 * @see <a href="http://www.w3schools.com/tags/ref_entities.asp">HTML Entity Reference</a>
+	 */
+	public boolean isFilterEntityReferences() {
+		return filterEntityReferences;
+	}
+
+	/**
+	 * Indicates that {@link #entityReference(String) entity references} should be filtered. Defaults to false. When
+	 * filtered, known HTML entity references are converted to their numeric counterpart, and unknown entity references
+	 * are emitted as plain text.
+	 * 
+	 * @since 1.6
+	 * @see <a href="http://www.w3schools.com/tags/ref_entities.asp">HTML Entity Reference</a>
+	 */
+	public void setFilterEntityReferences(boolean filterEntityReferences) {
+		this.filterEntityReferences = filterEntityReferences;
+	}
+
 }
diff --git a/org.eclipse.mylyn.wikitext.tests/src/org/eclipse/mylyn/wikitext/core/parser/builder/HtmlDocumentBuilderTest.java b/org.eclipse.mylyn.wikitext.tests/src/org/eclipse/mylyn/wikitext/core/parser/builder/HtmlDocumentBuilderTest.java
index 1afbe93e8..f0ddc4fd2 100644
--- a/org.eclipse.mylyn.wikitext.tests/src/org/eclipse/mylyn/wikitext/core/parser/builder/HtmlDocumentBuilderTest.java
+++ b/org.eclipse.mylyn.wikitext.tests/src/org/eclipse/mylyn/wikitext/core/parser/builder/HtmlDocumentBuilderTest.java
@@ -364,4 +364,32 @@ public class HtmlDocumentBuilderTest extends TestCase {
 
 		assertTrue(html.contains("<link type=\"text/css\" rel=\"stylesheet\" href=\"a/test.css\"/>"));
 	}
+
+	public void testFilterEntityReferences_FiltersKnown() {
+		builder.setFilterEntityReferences(true);
+
+		doEntityReferenceTest("yen", "&#165;");
+	}
+
+	public void testFilterEntityReferences_DisabledFilter() {
+		doEntityReferenceTest("yen", "&yen;");
+	}
+
+	public void testFilterEntityReferences_FiltersUnknown() {
+		builder.setFilterEntityReferences(true);
+
+		doEntityReferenceTest("unlikely", "&amp;unlikely;");
+	}
+
+	private void doEntityReferenceTest(String entityReference, String expected) {
+		builder.beginDocument();
+		builder.entityReference(entityReference);
+		builder.endDocument();
+
+		String html = out.toString();
+
+		TestUtil.println(html);
+
+		assertTrue(html.contains(expected));
+	}
 }
diff --git a/org.eclipse.mylyn.wikitext.ui/src/org/eclipse/mylyn/wikitext/ui/viewer/MarkupViewer.java b/org.eclipse.mylyn.wikitext.ui/src/org/eclipse/mylyn/wikitext/ui/viewer/MarkupViewer.java
index 9c6cb8d63..d9a9837ea 100644
--- a/org.eclipse.mylyn.wikitext.ui/src/org/eclipse/mylyn/wikitext/ui/viewer/MarkupViewer.java
+++ b/org.eclipse.mylyn.wikitext.ui/src/org/eclipse/mylyn/wikitext/ui/viewer/MarkupViewer.java
@@ -10,6 +10,8 @@
  *******************************************************************************/
 package org.eclipse.mylyn.wikitext.ui.viewer;
 
+import java.io.StringWriter;
+
 import org.eclipse.core.runtime.IStatus;
 import org.eclipse.jface.text.Document;
 import org.eclipse.jface.text.IDocument;
@@ -19,6 +21,7 @@ import org.eclipse.jface.text.source.IOverviewRuler;
 import org.eclipse.jface.text.source.IVerticalRuler;
 import org.eclipse.mylyn.internal.wikitext.ui.WikiTextUiPlugin;
 import org.eclipse.mylyn.wikitext.core.parser.MarkupParser;
+import org.eclipse.mylyn.wikitext.core.parser.builder.HtmlDocumentBuilder;
 import org.eclipse.mylyn.wikitext.core.parser.markup.MarkupLanguage;
 import org.eclipse.swt.widgets.Composite;
 
@@ -47,7 +50,14 @@ public class MarkupViewer extends HtmlViewer {
 
 	public void setMarkup(String source) {
 		try {
-			String htmlText = parser.parseToHtml(source);
+			StringWriter out = new StringWriter();
+			HtmlDocumentBuilder builder = new HtmlDocumentBuilder(out);
+			builder.setFilterEntityReferences(true);
+
+			parser.parse(source);
+			parser.setBuilder(null);
+
+			String htmlText = out.toString();
 			setHtml(htmlText);
 		} catch (Throwable t) {
 			if (getTextPresentation() != null) {