528344 - [Generic Editor] Default highlighter patterns are brokenI20180130-2000

Use a single (positive) pattern definition for valid words to match
both the current word and occurrences within the text

Change-Id: I6d05484b9991a543c2979005deae4533cb8c2735
Signed-off-by: Carsten Reckord <reckord@yatta.de>

4 files changed, 141 insertions, 17 deletions

diff --git a/org.eclipse.ui.genericeditor.tests/plugin.xml b/org.eclipse.ui.genericeditor.tests/plugin.xml
index a9c5904dc6d..1700b8ff825 100644
--- a/org.eclipse.ui.genericeditor.tests/plugin.xml
+++ b/org.eclipse.ui.genericeditor.tests/plugin.xml
@@ -140,7 +140,7 @@
 			point="org.eclipse.ui.genericeditor.highlightReconcilers">
 		<highlightReconciler
 			class="org.eclipse.ui.genericeditor.tests.contributions.HighlightReconciler"
-			contentType="org.eclipse.ui.genericeditor.tests.content-type">
+			contentType="org.eclipse.ui.genericeditor.tests.content-type-bar">
 		</highlightReconciler>
  </extension>
 </plugin>
diff --git a/org.eclipse.ui.genericeditor.tests/src/org/eclipse/ui/genericeditor/tests/AbstratGenericEditorTest.java b/org.eclipse.ui.genericeditor.tests/src/org/eclipse/ui/genericeditor/tests/AbstratGenericEditorTest.java
index ef1e3b0c4e5..b0a8635fada 100644
--- a/org.eclipse.ui.genericeditor.tests/src/org/eclipse/ui/genericeditor/tests/AbstratGenericEditorTest.java
+++ b/org.eclipse.ui.genericeditor.tests/src/org/eclipse/ui/genericeditor/tests/AbstratGenericEditorTest.java
@@ -55,8 +55,13 @@ public class AbstratGenericEditorTest {
 		project = ResourcesPlugin.getWorkspace().getRoot().getProject(getClass().getName() + System.currentTimeMillis());
 		project.create(null);
 		project.open(null);
-		createAndOpenFile("foo.txt", "bar 'bar'");
+		project.setDefaultCharset("UTF-8", null);
+		createAndOpenFile();
 	 }
+
+	protected void createAndOpenFile() throws Exception {
+		createAndOpenFile("foo.txt", "bar 'bar'");
+	}
 	
 	/**
 	 * Creates a new file in the project, opens it, and associate that file with the test state
@@ -67,7 +72,8 @@ public class AbstratGenericEditorTest {
 	 */
 	protected void createAndOpenFile(String name, String contents) throws Exception {
 		this.file = project.getFile(name);
-		this.file.create(new ByteArrayInputStream(contents.getBytes()), true, null);
+		this.file.create(new ByteArrayInputStream(contents.getBytes("UTF-8")), true, null);
+		this.file.setCharset("UTF-8", null);
 		this.editor = (ExtensionBasedTextEditor) PlatformUI.getWorkbench().getActiveWorkbenchWindow()
 				.getActivePage().openEditor(new FileEditorInput(this.file), "org.eclipse.ui.genericeditor.GenericEditor");
 	}
@@ -82,6 +88,7 @@ public class AbstratGenericEditorTest {
 			editor.close(false);
 			editor = null;
 		}
+		while(Display.getDefault().readAndDispatch()) {}
 		if (file != null) {
 			file.delete(true, new NullProgressMonitor());
 			file = null;
diff --git a/org.eclipse.ui.genericeditor.tests/src/org/eclipse/ui/genericeditor/tests/HighlightTest.java b/org.eclipse.ui.genericeditor.tests/src/org/eclipse/ui/genericeditor/tests/HighlightTest.java
index e259ffe307c..4d2c3657118 100644
--- a/org.eclipse.ui.genericeditor.tests/src/org/eclipse/ui/genericeditor/tests/HighlightTest.java
+++ b/org.eclipse.ui.genericeditor.tests/src/org/eclipse/ui/genericeditor/tests/HighlightTest.java
@@ -10,6 +10,8 @@
  *******************************************************************************/
 package org.eclipse.ui.genericeditor.tests;
 
+import static org.junit.Assert.assertEquals;
+
 import java.util.ArrayList;
 import java.util.Iterator;
 import java.util.List;
@@ -19,6 +21,8 @@ import org.junit.Test;
 
 import org.eclipse.swt.widgets.Display;
 
+import org.eclipse.jface.text.IDocument;
+import org.eclipse.jface.text.Position;
 import org.eclipse.jface.text.source.Annotation;
 import org.eclipse.jface.text.source.IAnnotationModel;
 import org.eclipse.jface.text.tests.util.DisplayHelper;
@@ -27,25 +31,115 @@ import org.eclipse.ui.texteditor.IDocumentProvider;
 
 public class HighlightTest extends AbstratGenericEditorTest {
 
-	private static final String ANNOTATION_TYPE = "org.eclipse.ui.genericeditor.text"; //$NON-NLS-1$
+	private static final String ANNOTATION_TYPE= "org.eclipse.ui.genericeditor.text"; //$NON-NLS-1$
+
+	private static final String EDITOR_TEXT= "Lorem ipsum dolor sit amet, consectetur adipiscing elit.\n" +
+			"Lorem ipsum dolor sit amet, consectetur adipiscing elit.\n" +
+			"Lorem ipsum dolor sit amet, consectetur adipiscing elit.";
+
+	@Override
+	protected void createAndOpenFile() throws Exception {
+		//leave editor creation to individual tests
+	}
+
+	@Test
+	public void testCustomHighlightReconciler() throws Exception {
+		createAndOpenFile("bar.txt", "bar 'bar'");
+
+		checkHighlightForCaretOffset(0, "'bar'", 1);
+	}
+
+	@Test
+	public void testHighlightWordAtDocumentStart() throws Exception {
+		createAndOpenFile("foo.txt", EDITOR_TEXT);
+
+		checkHighlightForCaretOffset(0, "Lorem", 3);
+	}
+
+	@Test
+	public void testHighlightWordWithNonLetterParts() throws Exception {
+		String complexWord= "dolor_sit123amet45";
+		String editorText= EDITOR_TEXT.replaceFirst("dolor sit amet", complexWord).replaceFirst("dolor sit amet", complexWord);
+		createAndOpenFile("foo.txt", editorText);
+
+		checkHighlightForCaretOffset(editorText.indexOf("dolor") + 3, complexWord, 2);
+		checkHighlightForCaretOffset(editorText.indexOf("_sit") + 3, complexWord, 2);
+		checkHighlightForCaretOffset(editorText.indexOf("123") + 1, complexWord, 2);
+		checkHighlightForCaretOffset(editorText.indexOf("amet") + 1, complexWord, 2);
+	}
+
+	@Test
+	public void testHighlightSimpleWordNotMatchingWordPart() throws Exception {
+		String complexWord= "dolor_sit123amet45";
+		String editorText= EDITOR_TEXT.replaceFirst("dolor sit amet", complexWord);
+		createAndOpenFile("foo.txt", editorText);
+
+		checkHighlightForCaretOffset(editorText.indexOf("dolor ") + 1, "dolor", 2);
+		checkHighlightForCaretOffset(editorText.indexOf(" sit") + 1, "sit", 2);
+		checkHighlightForCaretOffset(editorText.indexOf(" amet") + 1, "amet", 2);
+	}
 
 	@Test
-	public void testHighlightReconciler() {
-		IDocumentProvider dp = editor.getDocumentProvider();
-		IAnnotationModel am = dp.getAnnotationModel(editor.getEditorInput());
+	public void testHighlightNonAsciiCharacters() throws Exception {
+		String complexWord= "sit\u00f6\u00f6amet";
+		String editorText= EDITOR_TEXT.replaceFirst("sit amet", complexWord).replaceFirst("sit amet", complexWord);
+		createAndOpenFile("foo.txt", editorText);
+
+		checkHighlightForCaretOffset(editorText.indexOf("sit") + 1, complexWord, 2);
+		checkHighlightForCaretOffset(editorText.indexOf("\u00f6") + 1, complexWord, 2);
+		checkHighlightForCaretOffset(editorText.indexOf("amet") + 1, complexWord, 2);
+	}
+
+	private void checkHighlightForCaretOffset(int pos, String expectedHighlight, int expectedHighlightCount) throws Exception {
+		clearAnnotations();
+
+		editor.selectAndReveal(pos, 0);
+		waitForAnnotations(expectedHighlightCount);
+
+		List<Annotation> annotations= getAnnotationsFromAnnotationModel();
+
+		IAnnotationModel annotationModel= getAnnotationModel();
+		IDocument document= editor.getDocumentProvider().getDocument(editor.getEditorInput());
+		for (int i= 0; i < annotations.size(); i++) {
+			Annotation annotation= annotations.get(i);
+			Position position= annotationModel.getPosition(annotation);
+			String highlight= document.get(position.offset, position.length);
+			assertEquals("Wrong highlight " + i + " at position " + position.offset, expectedHighlight, highlight);
+		}
+		Assert.assertEquals("Wrong number of highlights", expectedHighlightCount, annotations.size());
+	}
+
+	private void clearAnnotations() {
+		editor.selectAndReveal(0, 0);
+		IAnnotationModel annotationModel= getAnnotationModel();
+		List<Annotation> annotations= getAnnotationsFromAnnotationModel();
+		for (Annotation annotation : annotations) {
+			annotationModel.removeAnnotation(annotation);
+		}
+	}
+
+	private IAnnotationModel getAnnotationModel() {
+		IDocumentProvider dp= editor.getDocumentProvider();
+		IAnnotationModel am= dp.getAnnotationModel(editor.getEditorInput());
+		return am;
+	}
+
+	private void waitForAnnotations(int count) {
 		new DisplayHelper() {
+			final IAnnotationModel annotationModel= getAnnotationModel();
+
 			@Override
 			protected boolean condition() {
-				return getAnnotationsFromAnnotationModel(am).size() == 1;
+				return getAnnotationsFromAnnotationModel().size() == count;
 			}
-		}.waitForCondition(Display.getDefault().getActiveShell().getDisplay(), 2000);
-		Assert.assertTrue("file does not have highlighting", getAnnotationsFromAnnotationModel(am).size() == 1);
+		}.waitForCondition(Display.getDefault(), 2000);
 	}
-	private List<Annotation> getAnnotationsFromAnnotationModel(IAnnotationModel annotationModel) {
-		List<Annotation> annotationList = new ArrayList<>();
-		Iterator<Annotation> annotationIterator=annotationModel.getAnnotationIterator();
+
+	private List<Annotation> getAnnotationsFromAnnotationModel() {
+		List<Annotation> annotationList= new ArrayList<>();
+		Iterator<Annotation> annotationIterator= getAnnotationModel().getAnnotationIterator();
 		while (annotationIterator.hasNext()) {
-			Annotation ann = annotationIterator.next();
+			Annotation ann= annotationIterator.next();
 			if (ann.getType().indexOf(ANNOTATION_TYPE) > -1) {
 				annotationList.add(ann);
 			}
diff --git a/org.eclipse.ui.genericeditor.tests/src/org/eclipse/ui/genericeditor/tests/ReconcilerTest.java b/org.eclipse.ui.genericeditor.tests/src/org/eclipse/ui/genericeditor/tests/ReconcilerTest.java
index 56c4a7cd43a..0f5470814a3 100644
--- a/org.eclipse.ui.genericeditor.tests/src/org/eclipse/ui/genericeditor/tests/ReconcilerTest.java
+++ b/org.eclipse.ui.genericeditor.tests/src/org/eclipse/ui/genericeditor/tests/ReconcilerTest.java
@@ -17,6 +17,8 @@ import org.junit.Test;
 
 import org.eclipse.swt.widgets.Display;
 
+import org.eclipse.core.runtime.NullProgressMonitor;
+
 import org.eclipse.core.resources.IFile;
 import org.eclipse.core.resources.IProject;
 import org.eclipse.core.resources.ResourcesPlugin;
@@ -36,6 +38,8 @@ import org.eclipse.ui.texteditor.IDocumentProvider;
 public class ReconcilerTest extends AbstratGenericEditorTest {
 
 	protected ExtensionBasedTextEditor secondEditor;
+	protected IFile secondFile;
+	protected IProject secondProject;
 
 	@Test
 	public void testReconciler() throws Exception {
@@ -44,10 +48,10 @@ public class ReconcilerTest extends AbstratGenericEditorTest {
 
 	@Test
 	public void testMultipleEditors() throws Exception {
-		IProject secondProject = ResourcesPlugin.getWorkspace().getRoot().getProject(getClass().getName() + System.currentTimeMillis());
+		secondProject= ResourcesPlugin.getWorkspace().getRoot().getProject(getClass().getName() + System.currentTimeMillis());
 		secondProject.create(null);
 		secondProject.open(null);
-		IFile secondFile = secondProject.getFile("foo.txt");
+		secondFile= secondProject.getFile("foo.txt");
 		secondFile.create(new ByteArrayInputStream("bar 'bar'".getBytes()), true, null);
 		secondEditor = (ExtensionBasedTextEditor) PlatformUI.getWorkbench().getActiveWorkbenchWindow()
 				.getActivePage().openEditor(new FileEditorInput(secondFile), "org.eclipse.ui.genericeditor.GenericEditor");
@@ -56,7 +60,7 @@ public class ReconcilerTest extends AbstratGenericEditorTest {
 	
 	@Test
 	public void testMultipleReconcilers() throws Exception {
-		IFile secondFile = project.getFile("bar.txt");
+		secondFile = project.getFile("bar.txt");
 		secondFile.create(new ByteArrayInputStream("".getBytes()), true, null);
 		secondEditor = (ExtensionBasedTextEditor) PlatformUI.getWorkbench().getActiveWorkbenchWindow()
 				.getActivePage().openEditor(new FileEditorInput(secondFile), "org.eclipse.ui.genericeditor.GenericEditor");
@@ -82,4 +86,23 @@ public class ReconcilerTest extends AbstratGenericEditorTest {
 		Assert.assertTrue("file was not affected by reconciler", doc.get().contains(expectedText));
 	}
 
+	@Override
+	public void tearDown() throws Exception {
+		if (secondEditor != null)
+		{
+			secondEditor.close(false);
+			secondEditor = null;
+			while(Display.getDefault().readAndDispatch()) {}
+		}
+		if (secondFile != null)
+		{
+			secondFile.delete(true, new NullProgressMonitor());
+			secondFile = null;
+		}
+		super.tearDown();
+		if (secondProject != null)
+		{
+			secondProject.delete(true, new NullProgressMonitor());
+		}
+	}
 }