513661: Enable ListBlocks within Confluence TableBlocks

Substantial refactor of TableBlock to allow nested ListBlocks in cell
content.
Currently limited to ListBlock nesting but in theory adding other Block
subclasses is possible in the future.

Change-Id: If1a6772281df128b1f3f34e7e823dbc34b356fea
Task-Url: https://bugs.eclipse.org/bugs/show_bug.cgi?id=513661
Signed-off-by: James Kennedy <james.kennedy@tasktop.com>

4 files changed, 213 insertions, 61 deletions

diff --git a/wikitext/core/org.eclipse.mylyn.wikitext.confluence/src/main/java/org/eclipse/mylyn/wikitext/confluence/internal/block/ListBlock.java b/wikitext/core/org.eclipse.mylyn.wikitext.confluence/src/main/java/org/eclipse/mylyn/wikitext/confluence/internal/block/ListBlock.java
index de5672269..7feddb0f9 100644
--- a/wikitext/core/org.eclipse.mylyn.wikitext.confluence/src/main/java/org/eclipse/mylyn/wikitext/confluence/internal/block/ListBlock.java
+++ b/wikitext/core/org.eclipse.mylyn.wikitext.confluence/src/main/java/org/eclipse/mylyn/wikitext/confluence/internal/block/ListBlock.java
@@ -1,5 +1,5 @@
 /*******************************************************************************
- * Copyright (c) 2007, 2012 David Green and others.
+ * Copyright (c) 2007, 2017 David Green and others.
  * All rights reserved. This program and the accompanying materials
  * are made available under the terms of the Eclipse Public License v1.0
  * which accompanies this distribution, and is available at
@@ -20,14 +20,14 @@ import org.eclipse.mylyn.wikitext.parser.markup.Block;
 
 /**
  * List block, matches blocks that start with <code>*</code>, <code>#</code> or <code>-</code>
- * 
+ *
  * @author David Green
  */
 public class ListBlock extends Block {
 
 	private static final int LINE_REMAINDER_GROUP_OFFSET = 2;
 
-	static final Pattern startPattern = Pattern.compile("((?:(?:\\*)|(?:#)|(?:-))+)\\s(.*+)"); //$NON-NLS-1$
+	private static final Pattern LIST_PATTERN = Pattern.compile("\\s*((?:(?:\\*)|(?:#)|(?:-))+)\\s(.*+)"); //$NON-NLS-1$
 
 	private int blockLineCount = 0;
 
@@ -60,7 +60,7 @@ public class ListBlock extends Block {
 
 			adjustLevel(listSpec, level, type);
 		} else {
-			Matcher matcher = startPattern.matcher(line);
+			Matcher matcher = LIST_PATTERN.matcher(line);
 			if (!matcher.matches()) {
 				boolean empty = offset == 0 && markupLanguage.isEmptyLine(line);
 				boolean breaking = ParagraphBlock.paragraphBreakingBlockMatches(getMarkupLanguage(), line, offset);
@@ -102,7 +102,8 @@ public class ListBlock extends Block {
 	}
 
 	private void adjustLevel(String listSpec, int level, BlockType type) {
-		for (ListState previousState = listState.peek(); level != previousState.level || previousState.type != type; previousState = listState.peek()) {
+		for (ListState previousState = listState.peek(); level != previousState.level
+				|| previousState.type != type; previousState = listState.peek()) {
 
 			if (level > previousState.level) {
 				if (!previousState.openItem) {
@@ -143,25 +144,21 @@ public class ListBlock extends Block {
 	public boolean canStart(String line, int lineOffset) {
 		blockLineCount = 0;
 		listState = null;
-		if (lineOffset == 0) {
-			matcher = startPattern.matcher(line);
-			final boolean matches = matcher.matches();
-			if (matches) {
-				String listSpec = matcher.group(1);
-				if (listSpec.charAt(0) == '-') {
-					int level = calculateLevel(listSpec);
-					if (level > 1) {
-						// don't match hr, emdash, endash etc.
-						// list block must start at level 1s
-						return false;
-					}
+		matcher = LIST_PATTERN.matcher(line);
+		matcher.region(lineOffset, line.length());
+		boolean matches = matcher.matches();
+		if (matches) {
+			String listSpec = matcher.group(1);
+			if (listSpec.charAt(0) == '-') {
+				int level = calculateLevel(listSpec);
+				if (level > 1) {
+					// don't match hr, emdash, endash etc.
+					// list block must start at level 1s
+					return false;
 				}
 			}
-			return matches;
-		} else {
-			matcher = null;
-			return false;
 		}
+		return matches;
 	}
 
 	@Override
diff --git a/wikitext/core/org.eclipse.mylyn.wikitext.confluence/src/main/java/org/eclipse/mylyn/wikitext/confluence/internal/block/TableBlock.java b/wikitext/core/org.eclipse.mylyn.wikitext.confluence/src/main/java/org/eclipse/mylyn/wikitext/confluence/internal/block/TableBlock.java
index fedb46ab3..34a1e115c 100644
--- a/wikitext/core/org.eclipse.mylyn.wikitext.confluence/src/main/java/org/eclipse/mylyn/wikitext/confluence/internal/block/TableBlock.java
+++ b/wikitext/core/org.eclipse.mylyn.wikitext.confluence/src/main/java/org/eclipse/mylyn/wikitext/confluence/internal/block/TableBlock.java
@@ -1,5 +1,5 @@
 /*******************************************************************************
- * Copyright (c) 2007, 2015 David Green and others.
+ * Copyright (c) 2007, 2017 David Green and others.
  * All rights reserved. This program and the accompanying materials
  * are made available under the terms of the Eclipse Public License v1.0
  * which accompanies this distribution, and is available at
@@ -10,14 +10,17 @@
  *******************************************************************************/
 package org.eclipse.mylyn.wikitext.confluence.internal.block;
 
+import java.util.List;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
+import org.eclipse.mylyn.wikitext.confluence.ConfluenceLanguage;
 import org.eclipse.mylyn.wikitext.parser.Attributes;
 import org.eclipse.mylyn.wikitext.parser.DocumentBuilder.BlockType;
 import org.eclipse.mylyn.wikitext.parser.markup.Block;
 
 import com.google.common.base.CharMatcher;
+import com.google.common.collect.ImmutableList;
 
 /**
  * Table block, matches blocks that start with <code>table. </code> or those that start with a table row.
@@ -26,20 +29,35 @@ import com.google.common.base.CharMatcher;
  */
 public class TableBlock extends Block {
 
-	static final Pattern startPattern = Pattern.compile("(\\|\\s*(.*)?(\\|\\s*$))"); //$NON-NLS-1$
+	private static final List<Class<?>> NESTABLE_CELL_BLOCKS = ImmutableList.of(ListBlock.class);
 
-	static final Pattern TABLE_ROW_PATTERN = Pattern.compile("\\|(\\|)?\\s*" + "((?:(?:[^\\|\\[]*)(?:\\[[^\\]]*\\])?)*)" //$NON-NLS-1$ //$NON-NLS-2$
-			+ "(\\|\\|?\\s*$)?"); //$NON-NLS-1$
+	private static final Pattern START_PATTERN = Pattern.compile("\\s*(\\|\\|?.*$)"); //$NON-NLS-1$
+
+	private static final Pattern END_OF_CELL_CONTENT_PATTERN = Pattern.compile("((?:(?:[^\\|\\[]*)(?:\\[[^\\]]*\\])?)*)" //$NON-NLS-1$
+			+ "(\\|\\|?\\s*)+?"); //$NON-NLS-1$
+
+	private static final Pattern END_OF_ROW_PATTERN = Pattern.compile("^\\|\\|?\\s*$"); //$NON-NLS-1$
+
+	private static final Pattern TABLE_ROW_PATTERN = Pattern
+			.compile("\\|(\\|)?\\s*" + "((?:(?:[^\\|\\[]*)(?:\\[[^\\]]*\\])?)*)" //$NON-NLS-1$ //$NON-NLS-2$
+					+ "(\\|\\|?\\s*$)?"); //$NON-NLS-1$
 
 	private int blockLineCount = 0;
 
 	private Matcher matcher;
 
+	private BlockType currentCell;
+
+	private boolean nesting = false;
+
+	private boolean rowStarted = false;
+
 	public TableBlock() {
 	}
 
 	@Override
 	public int processLineContent(String line, int offset) {
+		nesting = false;
 		if (blockLineCount == 0) {
 			Attributes attributes = new Attributes();
 			builder.beginBlock(BlockType.TABLE, attributes);
@@ -47,44 +65,89 @@ public class TableBlock extends Block {
 			setClosed(true);
 			return 0;
 		}
+
 		++blockLineCount;
 
-		if (offset == line.length()) {
+		if (atEndOfRow(line, offset)) {
+			ensureRowClosed();
 			return -1;
 		}
 
-		String textileLine = offset == 0 ? line : line.substring(offset);
-		Matcher rowMatcher = TABLE_ROW_PATTERN.matcher(textileLine);
-		if (!rowMatcher.find()) {
+		int postCellOffset = processCellContent(line, offset);
+		return isClosed() ? 0 : processEndOfLine(line, postCellOffset);
+	}
+
+	private boolean atEndOfRow(String line, int lineOffset) {
+		String restOfLine = line.substring(lineOffset);
+		return END_OF_ROW_PATTERN.matcher(restOfLine).find();
+	}
+
+	private int processCellContent(String line, int offset) {
+		int cellsOffset = 0;
+		String restOfline = offset == 0 ? line : line.substring(offset);
+		Matcher rowMatcher = TABLE_ROW_PATTERN.matcher(restOfline);
+		if (rowMatcher.find()) {
+			do {
+				ensureCellClosed();
+				cellsOffset = startNextCell(rowMatcher);
+				String cellContent = rowMatcher.group(2);
+				nesting = isNestableCellContent(cellContent);
+				if (!nesting) {
+					emitMarkup(cellContent, offset + cellsOffset);
+					cellsOffset = rowMatcher.end(2);
+				}
+			} while (!nesting && rowMatcher.find());
+		} else {
 			setClosed(true);
-			return 0;
 		}
 
-		builder.beginBlock(BlockType.TABLE_ROW, new Attributes());
-
-		do {
-			int start = rowMatcher.start();
-			if (start == textileLine.length() - 1) {
-				break;
-			}
+		return offset + cellsOffset;
+	}
 
-			String headerIndicator = rowMatcher.group(1);
-			String text = rowMatcher.group(2);
-			int lineOffset = offset + rowMatcher.start(2);
+	private int startNextCell(Matcher rowMatcher) {
+		ensureRowStarted();
+		String headerIndicator = rowMatcher.group(1);
+		boolean header = "|".equals(headerIndicator); //$NON-NLS-1$
+		currentCell = header ? BlockType.TABLE_CELL_HEADER : BlockType.TABLE_CELL_NORMAL;
+		builder.beginBlock(currentCell, new Attributes());
+		return rowMatcher.start(2);
+	}
 
-			boolean header = headerIndicator != null && "|".equals(headerIndicator); //$NON-NLS-1$
+	private boolean isNestableCellContent(String cellContent) {
+		return NESTABLE_CELL_BLOCKS.contains(getConfluenceLanguage().startBlock(cellContent, 0).getClass());
+	}
 
-			Attributes attributes = new Attributes();
-			builder.beginBlock(header ? BlockType.TABLE_CELL_HEADER : BlockType.TABLE_CELL_NORMAL, attributes);
+	private void emitMarkup(String text, int lineOffset) {
+		getConfluenceLanguage().emitMarkupLine(getParser(), state, lineOffset,
+				CharMatcher.WHITESPACE.trimTrailingFrom(text), 0);
+	}
 
-			markupLanguage.emitMarkupLine(getParser(), state, lineOffset,
-					CharMatcher.WHITESPACE.trimTrailingFrom(text), 0);
+	private ConfluenceLanguage getConfluenceLanguage() {
+		return (ConfluenceLanguage) getMarkupLanguage();
+	}
 
-			builder.endBlock(); // table cell
-		} while (rowMatcher.find());
+	private int processEndOfLine(String line, int offset) {
+		if (!nesting) {
+			ensureRowClosed();
+			return -1;
+		}
+		return offset >= line.length() ? -1 : offset;
+	}
 
-		builder.endBlock(); // table row
+	@Override
+	public boolean beginNesting() {
+		return nesting;
+	}
 
+	@Override
+	public int findCloseOffset(String line, int lineOffset) {
+		Matcher endMatcher = END_OF_CELL_CONTENT_PATTERN.matcher(line);
+		if (lineOffset != 0) {
+			endMatcher.region(lineOffset, line.length());
+		}
+		if (endMatcher.find()) {
+			return endMatcher.start(2);
+		}
 		return -1;
 	}
 
@@ -92,7 +155,7 @@ public class TableBlock extends Block {
 	public boolean canStart(String line, int lineOffset) {
 		blockLineCount = 0;
 		if (lineOffset == 0) {
-			matcher = startPattern.matcher(line);
+			matcher = START_PATTERN.matcher(line);
 			return matcher.matches();
 		} else {
 			matcher = null;
@@ -100,12 +163,34 @@ public class TableBlock extends Block {
 		}
 	}
 
+	private void ensureRowStarted() {
+		if (!rowStarted) {
+			builder.beginBlock(BlockType.TABLE_ROW, new Attributes());
+			rowStarted = true;
+		}
+	}
+
 	@Override
 	public void setClosed(boolean closed) {
 		if (closed && !isClosed()) {
+			ensureRowClosed();
 			builder.endBlock();
 		}
 		super.setClosed(closed);
 	}
 
+	private void ensureRowClosed() {
+		ensureCellClosed();
+		if (rowStarted) {
+			builder.endBlock();
+			rowStarted = false;
+		}
+	}
+
+	private void ensureCellClosed() {
+		if (currentCell != null) {
+			builder.endBlock();
+			currentCell = null;
+		}
+	}
 }
diff --git a/wikitext/core/org.eclipse.mylyn.wikitext.confluence/src/test/java/org/eclipse/mylyn/wikitext/confluence/ConfluenceLanguageIntegrationTest.java b/wikitext/core/org.eclipse.mylyn.wikitext.confluence/src/test/java/org/eclipse/mylyn/wikitext/confluence/ConfluenceLanguageIntegrationTest.java
index 78afc2548..ca7464344 100644
--- a/wikitext/core/org.eclipse.mylyn.wikitext.confluence/src/test/java/org/eclipse/mylyn/wikitext/confluence/ConfluenceLanguageIntegrationTest.java
+++ b/wikitext/core/org.eclipse.mylyn.wikitext.confluence/src/test/java/org/eclipse/mylyn/wikitext/confluence/ConfluenceLanguageIntegrationTest.java
@@ -31,6 +31,13 @@ public class ConfluenceLanguageIntegrationTest {
 		assertHtmlToConfluence(false);
 	}
 
+	@Test
+	// test for bug# 513661
+	public void listsInTables() {
+		String textile = "|* item 1\n* item 2|# item 3\n# item 4|";
+		assertRoundTrip(textile, textile);
+	}
+
 	private void assertHtmlToConfluence(boolean parseAsDocument) {
 		HtmlLanguage htmlLanguage = HtmlLanguage.builder()
 				.add(BlockType.PARAGRAPH)
@@ -45,4 +52,15 @@ public class ConfluenceLanguageIntegrationTest {
 
 		assertEquals("some text *bold here* more text\n\n", confluenceOut.toString());
 	}
+
+	private void assertRoundTrip(String textileIn, String textileOut) {
+		Writer confluenceOut = new StringWriter();
+		ConfluenceLanguage confluenceLanguage = new ConfluenceLanguage();
+
+		MarkupParser parser = new MarkupParser(confluenceLanguage);
+		parser.setBuilder(confluenceLanguage.createDocumentBuilder(confluenceOut));
+		parser.parse(textileIn, false);
+
+		assertEquals(textileOut, confluenceOut.toString().trim());
+	}
 }
diff --git a/wikitext/core/org.eclipse.mylyn.wikitext.confluence/src/test/java/org/eclipse/mylyn/wikitext/confluence/ConfluenceLanguageTest.java b/wikitext/core/org.eclipse.mylyn.wikitext.confluence/src/test/java/org/eclipse/mylyn/wikitext/confluence/ConfluenceLanguageTest.java
index 58ead6d44..3b59374e3 100644
--- a/wikitext/core/org.eclipse.mylyn.wikitext.confluence/src/test/java/org/eclipse/mylyn/wikitext/confluence/ConfluenceLanguageTest.java
+++ b/wikitext/core/org.eclipse.mylyn.wikitext.confluence/src/test/java/org/eclipse/mylyn/wikitext/confluence/ConfluenceLanguageTest.java
@@ -1,5 +1,5 @@
 /*******************************************************************************
- * Copyright (c) 2007, 2015 David Green and others.
+ * Copyright (c) 2007, 2017 David Green and others.
  * All rights reserved. This program and the accompanying materials
  * are made available under the terms of the Eclipse Public License v1.0
  * which accompanies this distribution, and is available at
@@ -652,25 +652,21 @@ public class ConfluenceLanguageTest extends AbstractMarkupGenerationTest<Conflue
 
 	@Test
 	public void testTable() {
-		String html = parser.parseToHtml("|a|row|not header|");
-
-		assertTrue(html.contains("<body><table><tr><td>a</td><td>row</td><td>not header</td></tr></table></body>"));
+		assertMarkup("<table><tr><td>a</td><td>row</td><td>not header</td></tr></table>", "|a|row|not header|");
 	}
 
 	@Test
 	public void testTableWithHeader() {
-		String html = parser.parseToHtml("||a||header||row||\n|a|row|not header|");
-
-		assertTrue(html.contains(
-				"<body><table><tr><th>a</th><th>header</th><th>row</th></tr><tr><td>a</td><td>row</td><td>not header</td></tr></table></body>"));
+		assertMarkup(
+				"<table><tr><th>a</th><th>header</th><th>row</th></tr><tr><td>a</td><td>row</td><td>not header</td></tr></table>",
+				"||a||header||row||\n|a|row|not header|");
 	}
 
 	@Test
 	public void testTableNestedWithHeader() {
-		String html = parser.parseToHtml("a para\n||a||header||row||\n|a|row|not header|\ntail");
-
-		assertTrue(html.contains(
-				"<body><p>a para</p><table><tr><th>a</th><th>header</th><th>row</th></tr><tr><td>a</td><td>row</td><td>not header</td></tr></table><p>tail</p></body>"));
+		assertMarkup(
+				"<p>a para</p><table><tr><th>a</th><th>header</th><th>row</th></tr><tr><td>a</td><td>row</td><td>not header</td></tr></table><p>tail</p>",
+				"a para\n||a||header||row||\n|a|row|not header|\ntail");
 	}
 
 	@Test
@@ -689,6 +685,57 @@ public class ConfluenceLanguageTest extends AbstractMarkupGenerationTest<Conflue
 	}
 
 	@Test
+	public void testTableWithSingletonList() {
+		// test for bug# 513661
+		assertMarkup("<table><tr><td><ul><li>one thing</li></ul></td><td>another cell</td></tr></table>",
+				"|* one thing| another cell |");
+	}
+
+	@Test
+	public void testTableWithSingletonListAndWhitespacePrefix() {
+		// test for bug# 513661
+		assertMarkup("<table><tr><td><ul><li>one thing</li></ul></td><td>another cell</td></tr></table>",
+				"|    * one thing| another cell |");
+	}
+
+	@Test
+	public void testTableWithBulletedLists() {
+		// test for bug# 513661
+		assertMarkup(
+				"<table><tr><td><ul><li>one thing</li><li>two things</li></ul></td><td>another cell</td></tr></table>",
+				"|* one thing\n* two things| another cell |");
+	}
+
+	@Test
+	public void testTableWithNumberedLists() {
+		// test for bug# 513661
+		assertMarkup(
+				"<table><tr><td>other cell</td><td><ol><li>one thing</li><li>two things</li><li>three things </li></ol></td></tr></table>",
+				"|other cell| # one thing\n# two things\n# three things |");
+	}
+
+	@Test
+	public void testTableWithLinksAndLists() {
+		// test for bug# 513661
+		assertMarkup(
+				"<table><tr><td><a href=\"https://textile-j.dev.java.net/\">Website</a></td><td><ol><li>one thing</li><li>two things</li><li>three things </li></ol></td><td><a href=\"http://www.eclipse.org\">Eclipse</a></td></tr></table>",
+				"| [Website|https://textile-j.dev.java.net/]| # one thing\n# two things\n# three things | [Eclipse|http://www.eclipse.org] |");
+	}
+
+	@Test
+	public void testTableWithMultipleLists() {
+		// test for bug# 513661
+		assertMarkup("<table>" + //
+				"<tr><th>Bulleted list</th><th><ul><li>one thing</li><li>two things </li></ul></th></tr>" + //
+				"<tr><td>Numbered list</td><td><ol><li>one thing</li><li>two things </li></ol></td></tr>" + //
+				"<tr><td>Bulleted list</td><td><ul style=\"list-style: square\"><li>one thing<ul><li>two things </li></ul></li></ul></td></tr>"
+				+ //
+				"</table>", "||Bulleted list||* one thing\n* two things |\n" + //
+						"|Numbered list|# one thing\n# two things |\n" + //
+						"|Bulleted list|- one thing\n-- two things |");
+	}
+
+	@Test
 	public void testPreformattedExtended() {
 		String html = parser
 				.parseToHtml("{noformat}\na multiline\n\tpreformatted\n\nwith two paras\n{noformat}\nanother para");
@@ -1066,6 +1113,11 @@ public class ConfluenceLanguageTest extends AbstractMarkupGenerationTest<Conflue
 	}
 
 	@Test
+	public void testListItemWithIndent() {
+		assertMarkup("<ul><li>one<br/>two</li><li>three</li></ul>", " \t* one\ntwo\n* three");
+	}
+
+	@Test
 	public void testListItemWithTwoNewlines() {
 		String html = parser.parseToHtml("* one\n\ntwo\n* three");