bug 191522: provide full text search functionality over task comments

https://bugs.eclipse.org/bugs/show_bug.cgi?id=191522

Make it possible for connectors to specify what they want indexed as
content.
Fix bug in cached match hits after reindexing.
Fix bug in closing of file handles.

2 files changed, 100 insertions, 8 deletions

diff --git a/org.eclipse.mylyn.tasks.index.core/src/org/eclipse/mylyn/internal/tasks/index/core/TaskListIndex.java b/org.eclipse.mylyn.tasks.index.core/src/org/eclipse/mylyn/internal/tasks/index/core/TaskListIndex.java
index 57343c36..2fae2b59 100644
--- a/org.eclipse.mylyn.tasks.index.core/src/org/eclipse/mylyn/internal/tasks/index/core/TaskListIndex.java
+++ b/org.eclipse.mylyn.tasks.index.core/src/org/eclipse/mylyn/internal/tasks/index/core/TaskListIndex.java
@@ -15,10 +15,12 @@ import java.io.File;
 import java.io.FileNotFoundException;

 import java.io.IOException;

 import java.util.ArrayList;

+import java.util.Collection;

 import java.util.Date;

 import java.util.HashMap;

 import java.util.HashSet;

 import java.util.Iterator;

+import java.util.LinkedHashSet;

 import java.util.List;

 import java.util.Map;

 import java.util.Map.Entry;

@@ -88,6 +90,13 @@ public class TaskListIndex implements ITaskDataManagerListener, ITaskListChangeL
 

 	private static final Object COMMAND_RESET_INDEX = "index:reset"; //$NON-NLS-1$

 

+	/**

+	 * Task attribute meta-data key that should be set to "true" to have attribute value indexed as part of the task

+	 * {@link IndexField#CONTENT}. Provides a way for connectors to specify non-standard attributes as indexable. By

+	 * default, {@link TaskAttribute#SUMMARY summary} and {@link TaskAttribute#DESCRIPTION description} are indexed.

+	 */

+	public static final String META_INDEXED_AS_CONTENT = "index-content"; //$NON-NLS-1$

+

 	public static enum IndexField {

 		IDENTIFIER(false, null, false), //

 		TASK_KEY(false, null, false), //

@@ -253,7 +262,9 @@ public class TaskListIndex implements ITaskDataManagerListener, ITaskListChangeL
 

 	public void setDefaultField(IndexField defaultField) {

 		this.defaultField = defaultField;

-		lastResults = null;

+		synchronized (this) {

+			lastResults = null;

+		}

 	}

 

 	public int getMaxMatchSearchHits() {

@@ -317,7 +328,6 @@ public class TaskListIndex implements ITaskDataManagerListener, ITaskListChangeL
 			Set<String> hits;

 

 			synchronized (indexReader) {

-

 				if (lastResults == null || (lastPatternString == null || !lastPatternString.equals(patternString))) {

 					this.lastPatternString = patternString;

 

@@ -333,7 +343,6 @@ public class TaskListIndex implements ITaskDataManagerListener, ITaskListChangeL
 							Document document = indexReader.document(scoreDoc.doc);

 							hits.add(document.get(IndexField.IDENTIFIER.fieldName()));

 						}

-						lastResults = hits;

 					} catch (IOException e) {

 						StatusHandler.fail(new Status(IStatus.ERROR, TasksIndexCore.BUNDLE_ID,

 								"Unexpected failure within task list index", e)); //$NON-NLS-1$

@@ -351,6 +360,12 @@ public class TaskListIndex implements ITaskDataManagerListener, ITaskListChangeL
 					hits = lastResults;

 				}

 			}

+			synchronized (this) {

+				if (this.indexReader == indexReader) {

+					this.lastPatternString = patternString;

+					this.lastResults = hits;

+				}

+			}

 			String taskIdentifier = task.getHandleIdentifier();

 			return hits != null && hits.contains(taskIdentifier);

 		}

@@ -477,6 +492,7 @@ public class TaskListIndex implements ITaskDataManagerListener, ITaskListChangeL
 			synchronized (this) {

 				if (indexReader == null) {

 					indexReader = IndexReader.open(directory, true);

+					lastResults = null;

 				}

 				return indexReader;

 			}

@@ -534,9 +550,10 @@ public class TaskListIndex implements ITaskDataManagerListener, ITaskListChangeL
 		addIndexedAttribute(document, IndexField.TASK_KEY, task.getTaskKey());

 		addIndexedAttribute(document, IndexField.REPOSITORY_URL, task.getRepositoryUrl());

 		addIndexedAttribute(document, IndexField.SUMMARY, root.getMappedAttribute(TaskAttribute.SUMMARY));

-		addIndexedAttribute(document, IndexField.CONTENT, root.getMappedAttribute(TaskAttribute.SUMMARY));

-		addIndexedAttribute(document, IndexField.CONTENT, root.getMappedAttribute(TaskAttribute.DESCRIPTION));

-		addIndexedAttribute(document, IndexField.CONTENT, root.getAttribute("status_whiteboard")); //$NON-NLS-1$

+

+		for (TaskAttribute contentAttribute : computeContentAttributes(root)) {

+			addIndexedAttribute(document, IndexField.CONTENT, contentAttribute);

+		}

 

 		addIndexedDateAttributes(document, task);

 

@@ -569,6 +586,38 @@ public class TaskListIndex implements ITaskDataManagerListener, ITaskListChangeL
 		}

 	}

 

+	/**

+	 * compute attributes that should be indexed as {@link IndexField#CONTENT}

+	 */

+	private Collection<TaskAttribute> computeContentAttributes(TaskAttribute root) {

+		Set<TaskAttribute> attributes = new LinkedHashSet<TaskAttribute>();

+

+		// add default content attributes

+		{

+			TaskAttribute attribute = root.getMappedAttribute(TaskAttribute.SUMMARY);

+			if (attribute != null) {

+				attributes.add(attribute);

+			}

+			attribute = root.getMappedAttribute(TaskAttribute.DESCRIPTION);

+			if (attribute != null) {

+				attributes.add(attribute);

+			}

+			// bugzilla

+			attribute = root.getAttribute("status_whiteboard"); //$NON-NLS-1$

+			if (attribute != null) {

+				attributes.add(attribute);

+			}

+		}

+

+		for (TaskAttribute attribute : root.getAttributes().values()) {

+			if (Boolean.parseBoolean(attribute.getMetaData().getValue(META_INDEXED_AS_CONTENT))) {

+				attributes.add(attribute);

+			}

+		}

+

+		return attributes;

+	}

+

 	private void addIndexedAttributes(Document document, ITask task) {

 		addIndexedAttribute(document, IndexField.TASK_KEY, task.getTaskKey());

 		addIndexedAttribute(document, IndexField.REPOSITORY_URL, task.getRepositoryUrl());

@@ -754,7 +803,12 @@ public class TaskListIndex implements ITaskDataManagerListener, ITaskListChangeL
 						monitor.worked(WORK_PER_SEGMENT);

 					}

 					synchronized (TaskListIndex.this) {

-						indexReader = null;

+						if (indexReader != null) {

+							synchronized (indexReader) {

+								indexReader.close();

+							}

+							indexReader = null;

+						}

 					}

 				} catch (CoreException e) {

 					throw e;

diff --git a/org.eclipse.mylyn.tasks.index.tests/src/org/eclipse/mylyn/internal/tasks/index/tests/TaskListIndexTest.java b/org.eclipse.mylyn.tasks.index.tests/src/org/eclipse/mylyn/internal/tasks/index/tests/TaskListIndexTest.java
index 2be00fed..ee82c2f5 100644
--- a/org.eclipse.mylyn.tasks.index.tests/src/org/eclipse/mylyn/internal/tasks/index/tests/TaskListIndexTest.java
+++ b/org.eclipse.mylyn.tasks.index.tests/src/org/eclipse/mylyn/internal/tasks/index/tests/TaskListIndexTest.java
@@ -24,6 +24,7 @@ import java.util.Date;
 import java.util.HashSet;

 import java.util.List;

 import java.util.Set;

+import java.util.logging.Logger;

 

 import org.eclipse.core.runtime.CoreException;

 import org.eclipse.mylyn.commons.core.DelegatingProgressMonitor;

@@ -103,7 +104,9 @@ public class TaskListIndexTest {
 				}

 			}

 		}

-		file.delete();

+		if (!file.delete()) {

+			Logger.getLogger(TaskListIndexTest.class.getName()).severe("Cannot delete: " + file);

+		}

 	}

 

 	private void setupIndex() {

@@ -280,4 +283,39 @@ public class TaskListIndexTest {
 			assertEquals("a\\" + special + "b", index.escapeFieldValue("a" + special + "b"));

 		}

 	}

+

+	@Test

+	public void testAttributeMetadataAffectsIndexing() throws CoreException, InterruptedException {

+		setupIndex();

+

+		ITask repositoryTask = context.createRepositoryTask();

+

+		index.waitUntilIdle();

+		index.setDefaultField(IndexField.CONTENT);

+

+		TaskData taskData = context.getDataManager().getTaskData(repositoryTask);

+

+		// sanity

+		assertNotNull(taskData);

+

+		final String content = "c" + System.currentTimeMillis();

+

+		// setup data so that it will match

+		TaskAttribute attribute = taskData.getRoot().createAttribute("unusualIndexedAttribute");

+		attribute.setValue(content);

+

+		// update

+		context.getDataManager().putSubmittedTaskData(repositoryTask, taskData, new DelegatingProgressMonitor());

+

+		// verify index doesn't match search term

+		assertFalse(index.matches(repositoryTask, content));

+

+		// now make data indexable

+		attribute.getMetaData().putValue(TaskListIndex.META_INDEXED_AS_CONTENT, "true");

+		// update

+		context.getDataManager().putSubmittedTaskData(repositoryTask, taskData, new DelegatingProgressMonitor());

+

+		// should now match

+		assertTrue(index.matches(repositoryTask, content));

+	}

 }