diff options
Diffstat (limited to 'org.eclipse.mylyn.tasks.index.core/src')
5 files changed, 1049 insertions, 0 deletions
diff --git a/org.eclipse.mylyn.tasks.index.core/src/org/eclipse/mylyn/internal/tasks/index/core/Messages.java b/org.eclipse.mylyn.tasks.index.core/src/org/eclipse/mylyn/internal/tasks/index/core/Messages.java new file mode 100644 index 000000000..af9c99b1b --- /dev/null +++ b/org.eclipse.mylyn.tasks.index.core/src/org/eclipse/mylyn/internal/tasks/index/core/Messages.java @@ -0,0 +1,32 @@ +/******************************************************************************* + * Copyright (c) 2011 Tasktop Technologies and others. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * Contributors: + * Tasktop Technologies - initial API and implementation + *******************************************************************************/ + +package org.eclipse.mylyn.internal.tasks.index.core; + +import org.eclipse.osgi.util.NLS; + +/** + * @author David Green + */ +class Messages extends NLS { + private static final String BUNDLE_NAME = "org.eclipse.mylyn.internal.tasks.index.core.messages"; //$NON-NLS-1$ + + public static String TaskListIndex_indexerJob; + + public static String TaskListIndex_task_rebuildingIndex; + static { + // initialize resource bundle + NLS.initializeMessages(BUNDLE_NAME, Messages.class); + } + + private Messages() { + } +} diff --git a/org.eclipse.mylyn.tasks.index.core/src/org/eclipse/mylyn/internal/tasks/index/core/TaskAnalyzer.java b/org.eclipse.mylyn.tasks.index.core/src/org/eclipse/mylyn/internal/tasks/index/core/TaskAnalyzer.java new file mode 100644 index 000000000..297bca42b --- /dev/null +++ b/org.eclipse.mylyn.tasks.index.core/src/org/eclipse/mylyn/internal/tasks/index/core/TaskAnalyzer.java @@ -0,0 +1,33 @@ +/******************************************************************************* + * Copyright (c) 2011 Tasktop Technologies. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * Contributors: + * Tasktop Technologies - initial API and implementation + *******************************************************************************/ + +package org.eclipse.mylyn.internal.tasks.index.core; + +import org.apache.lucene.analysis.KeywordAnalyzer; +import org.apache.lucene.analysis.PerFieldAnalyzerWrapper; +import org.apache.lucene.analysis.standard.StandardAnalyzer; +import org.apache.lucene.util.Version; + +/** + * An analyzer that is aware of task fields + * + * @author David Green + */ +class TaskAnalyzer extends PerFieldAnalyzerWrapper { + + public TaskAnalyzer() { + super(new StandardAnalyzer(Version.LUCENE_CURRENT)); + addAnalyzer(TaskListIndex.IndexField.IDENTIFIER.fieldName(), new KeywordAnalyzer()); + addAnalyzer(TaskListIndex.IndexField.TASK_KEY.fieldName(), new KeywordAnalyzer()); + addAnalyzer(TaskListIndex.IndexField.REPOSITORY_URL.fieldName(), new KeywordAnalyzer()); + } + +} diff --git a/org.eclipse.mylyn.tasks.index.core/src/org/eclipse/mylyn/internal/tasks/index/core/TaskListIndex.java b/org.eclipse.mylyn.tasks.index.core/src/org/eclipse/mylyn/internal/tasks/index/core/TaskListIndex.java new file mode 100644 index 000000000..f778f69e9 --- /dev/null +++ b/org.eclipse.mylyn.tasks.index.core/src/org/eclipse/mylyn/internal/tasks/index/core/TaskListIndex.java @@ -0,0 +1,953 @@ +/******************************************************************************* + * Copyright (c) 2011 Tasktop Technologies. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * Contributors: + * Tasktop Technologies - initial API and implementation + *******************************************************************************/ + +package org.eclipse.mylyn.internal.tasks.index.core; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; +import java.util.logging.Logger; + +import org.apache.lucene.document.DateTools; +import org.apache.lucene.document.DateTools.Resolution; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.Field.Store; +import org.apache.lucene.index.CorruptIndexException; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.Term; +import org.apache.lucene.queryParser.ParseException; +import org.apache.lucene.queryParser.QueryParser; +import org.apache.lucene.search.BooleanClause; +import org.apache.lucene.search.BooleanClause.Occur; +import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.PrefixQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.ScoreDoc; +import org.apache.lucene.search.TermQuery; +import org.apache.lucene.search.TopDocs; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.NIOFSDirectory; +import org.apache.lucene.util.Version; +import org.eclipse.core.runtime.CoreException; +import org.eclipse.core.runtime.IProgressMonitor; +import org.eclipse.core.runtime.IStatus; +import org.eclipse.core.runtime.MultiStatus; +import org.eclipse.core.runtime.NullProgressMonitor; +import org.eclipse.core.runtime.Platform; +import org.eclipse.core.runtime.Status; +import org.eclipse.core.runtime.SubMonitor; +import org.eclipse.core.runtime.jobs.Job; +import org.eclipse.mylyn.commons.core.StatusHandler; +import org.eclipse.mylyn.internal.tasks.core.AbstractTask; +import org.eclipse.mylyn.internal.tasks.core.ITaskListChangeListener; +import org.eclipse.mylyn.internal.tasks.core.ITaskListRunnable; +import org.eclipse.mylyn.internal.tasks.core.TaskContainerDelta; +import org.eclipse.mylyn.internal.tasks.core.TaskList; +import org.eclipse.mylyn.internal.tasks.core.data.ITaskDataManagerListener; +import org.eclipse.mylyn.internal.tasks.core.data.TaskDataManager; +import org.eclipse.mylyn.internal.tasks.core.data.TaskDataManagerEvent; +import org.eclipse.mylyn.tasks.core.IRepositoryElement; +import org.eclipse.mylyn.tasks.core.IRepositoryPerson; +import org.eclipse.mylyn.tasks.core.ITask; +import org.eclipse.mylyn.tasks.core.data.TaskAttribute; +import org.eclipse.mylyn.tasks.core.data.TaskCommentMapper; +import org.eclipse.mylyn.tasks.core.data.TaskData; + +/** + * An index on a task list. + * + * @author David Green + */ +public class TaskListIndex implements ITaskDataManagerListener, ITaskListChangeListener { + + public abstract static class TaskCollector { + + public abstract void collect(ITask task); + + } + + private static final Object COMMAND_RESET_INDEX = "index:reset"; //$NON-NLS-1$ + + /** + * Task attribute meta-data key that should be set to "true" to have attribute value indexed as part of the task + * {@link IndexField#CONTENT}. Provides a way for connectors to specify non-standard attributes as indexable. By + * default, {@link TaskAttribute#SUMMARY summary} and {@link TaskAttribute#DESCRIPTION description} are indexed. + */ + public static final String META_INDEXED_AS_CONTENT = "index-content"; //$NON-NLS-1$ + + public static enum IndexField { + IDENTIFIER(false, null, false), // + TASK_KEY(false, null, false), // + REPOSITORY_URL(false, null, false), // + SUMMARY(true, null, false), // + CONTENT(true, null, false), // + ASSIGNEE(true, TaskAttribute.USER_ASSIGNED, false), // + REPORTER(true, TaskAttribute.USER_REPORTER, false), // + PERSON(true, null, false), // + COMPONENT(true, TaskAttribute.COMPONENT, false), // + COMPLETION_DATE(true, null, true), // + CREATION_DATE(true, null, true), // + DUE_DATE(true, null, true), // + MODIFICATION_DATE(true, null, true), // + DESCRIPTION(true, TaskAttribute.DESCRIPTION, false), // + KEYWORDS(true, TaskAttribute.KEYWORDS, false), // + PRODUCT(true, TaskAttribute.PRODUCT, false), // + RESOLUTION(true, TaskAttribute.RESOLUTION, false), // + SEVERITY(true, TaskAttribute.SEVERITY, false), // + STATUS(true, TaskAttribute.STATUS, false); + + private final boolean userVisible; + + private final String attributeId; + + private final boolean dateTime; + + private IndexField(boolean userVisible, String attributeId, boolean dateTime) { + this.userVisible = userVisible; + this.attributeId = attributeId; + this.dateTime = dateTime; + } + + public String fieldName() { + return name().toLowerCase(); + } + + /** + * get the task attribute id, or null if this field has special handling + */ + public String getAttributeId() { + return attributeId; + } + + /** + * indicate if the field should be exposed in the UI + */ + public boolean isUserVisible() { + return userVisible; + } + + /** + * indicate if the field is a date/time field + */ + public boolean isDateTime() { + return dateTime; + } + + public static IndexField fromFieldName(String fieldName) { + try { + return IndexField.valueOf(fieldName.toUpperCase()); + } catch (IllegalArgumentException e) { + return null; + } + } + + public boolean isPersonField() { + return this == PERSON || this == REPORTER || this == ASSIGNEE; + } + } + + private static enum MaintainIndexType { + STARTUP, REINDEX + } + + private Directory directory; + + private MaintainIndexJob maintainIndexJob; + + private final Map<ITask, TaskData> reindexQueue = new HashMap<ITask, TaskData>(); + + private IndexReader indexReader; + + private boolean rebuildIndex = false; + + private String lastPatternString; + + private Set<String> lastResults; + + private IndexField defaultField = IndexField.SUMMARY; + + private final TaskList taskList; + + private final TaskDataManager dataManager; + + private long startupDelay = 6000L; + + private long reindexDelay = 3000L; + + private int maxMatchSearchHits = 1500; + + private final ReadWriteLock indexReaderLock = new ReentrantReadWriteLock(true); + + private TaskListIndex(TaskList taskList, TaskDataManager dataManager) { + if (taskList == null) { + throw new IllegalArgumentException(); + } + if (dataManager == null) { + throw new IllegalArgumentException(); + } + this.taskList = taskList; + this.dataManager = dataManager; + } + + public TaskListIndex(TaskList taskList, TaskDataManager dataManager, File indexLocation) { + this(taskList, dataManager, indexLocation, 6000L); + } + + public TaskListIndex(TaskList taskList, TaskDataManager dataManager, File indexLocation, long startupDelay) { + this(taskList, dataManager); + if (startupDelay < 0L || startupDelay > (1000L * 60)) { + throw new IllegalArgumentException(); + } + if (indexLocation == null) { + throw new IllegalArgumentException(); + } + this.startupDelay = startupDelay; + if (!indexLocation.exists()) { + rebuildIndex = true; + if (!indexLocation.mkdirs()) { + StatusHandler.log(new Status(IStatus.ERROR, TasksIndexCore.BUNDLE_ID, + "Cannot create task list index folder: " + indexLocation)); //$NON-NLS-1$ + } + } + if (indexLocation.exists() && indexLocation.isDirectory()) { + try { + Logger.getLogger(TaskListIndex.class.getName()).fine("task list index: " + indexLocation); //$NON-NLS-1$ + + directory = new NIOFSDirectory(indexLocation); + } catch (IOException e) { + StatusHandler.log(new Status(IStatus.ERROR, TasksIndexCore.BUNDLE_ID, + "Cannot create task list index", e)); //$NON-NLS-1$ + } + } + initialize(); + } + + public TaskListIndex(TaskList taskList, TaskDataManager dataManager, Directory directory) { + this(taskList, dataManager); + this.directory = directory; + initialize(); + } + + public long getReindexDelay() { + return reindexDelay; + } + + public void setReindexDelay(long reindexDelay) { + this.reindexDelay = reindexDelay; + } + + public IndexField getDefaultField() { + return defaultField; + } + + public void setDefaultField(IndexField defaultField) { + this.defaultField = defaultField; + synchronized (this) { + lastResults = null; + } + } + + public int getMaxMatchSearchHits() { + return maxMatchSearchHits; + } + + public void setMaxMatchSearchHits(int maxMatchSearchHits) { + this.maxMatchSearchHits = maxMatchSearchHits; + } + + private void initialize() { + if (!rebuildIndex) { + IndexReader indexReader = null; + try { + indexReader = getIndexReader(); + } catch (Exception e) { + // ignore, this can happen if the index is corrupt + } + if (indexReader == null) { + rebuildIndex = true; + } + } + maintainIndexJob = new MaintainIndexJob(); + dataManager.addListener(this); + taskList.addChangeListener(this); + + scheduleIndexMaintenance(MaintainIndexType.STARTUP); + } + + private void scheduleIndexMaintenance(MaintainIndexType type) { + long delay = 0L; + switch (type) { + case STARTUP: + delay = startupDelay; + break; + case REINDEX: + delay = reindexDelay; + } + + if (delay == 0L) { + // primarily for testing purposes + + maintainIndexJob.cancel(); + try { + maintainIndexJob.join(); + } catch (InterruptedException e) { + // ignore + } + maintainIndexJob.run(new NullProgressMonitor()); + } else { + maintainIndexJob.schedule(delay); + } + } + + public boolean matches(ITask task, String patternString) { + if (patternString.equals(COMMAND_RESET_INDEX)) { + reindex(); + } + Lock readLock = indexReaderLock.readLock(); + readLock.lock(); + try { + + IndexReader indexReader = getIndexReader(); + if (indexReader != null) { + Set<String> hits; + + if (lastResults == null || (lastPatternString == null || !lastPatternString.equals(patternString))) { + this.lastPatternString = patternString; + + long startTime = System.currentTimeMillis(); + + hits = new HashSet<String>(); + + IndexSearcher indexSearcher = new IndexSearcher(indexReader); + try { + Query query = computeQuery(patternString); + TopDocs results = indexSearcher.search(query, maxMatchSearchHits); + for (ScoreDoc scoreDoc : results.scoreDocs) { + Document document = indexReader.document(scoreDoc.doc); + hits.add(document.get(IndexField.IDENTIFIER.fieldName())); + } + } catch (IOException e) { + StatusHandler.log(new Status(IStatus.ERROR, TasksIndexCore.BUNDLE_ID, + "Unexpected failure within task list index", e)); //$NON-NLS-1$ + } finally { + try { + indexSearcher.close(); + } catch (IOException e) { + // ignore + } + } + + Logger.getLogger(TaskListIndex.class.getName()).fine( + "New query in " + (System.currentTimeMillis() - startTime) + "ms"); //$NON-NLS-1$ //$NON-NLS-2$ + } else { + hits = lastResults; + } + synchronized (this) { + if (this.indexReader == indexReader) { + this.lastPatternString = patternString; + this.lastResults = hits; + } + } + String taskIdentifier = task.getHandleIdentifier(); + return hits != null && hits.contains(taskIdentifier); + } + + } finally { + readLock.unlock(); + } + return false; + } + + public void reindex() { + rebuildIndex = true; + scheduleIndexMaintenance(MaintainIndexType.REINDEX); + } + + /** + * call to wait until index maintenance has completed + * + * @throws InterruptedException + */ + public void waitUntilIdle() throws InterruptedException { + if (!Platform.isRunning() && reindexDelay != 0L) { + // job join() behaviour is not the same when platform is not running + Logger.getLogger(TaskListIndex.class.getName()).warning( + "Index job joining may not work properly when Eclipse platform is not running"); //$NON-NLS-1$ + } + maintainIndexJob.join(); + } + + public void find(String patternString, TaskCollector collector, int resultsLimit) { + + Lock readLock = indexReaderLock.readLock(); + readLock.lock(); + try { + IndexReader indexReader = getIndexReader(); + if (indexReader != null) { + IndexSearcher indexSearcher = new IndexSearcher(indexReader); + try { + Query query = computeQuery(patternString); + TopDocs results = indexSearcher.search(query, resultsLimit); + for (ScoreDoc scoreDoc : results.scoreDocs) { + Document document = indexReader.document(scoreDoc.doc); + String taskIdentifier = document.get(IndexField.IDENTIFIER.fieldName()); + AbstractTask task = taskList.getTask(taskIdentifier); + if (task != null) { + collector.collect(task); + } + } + } catch (IOException e) { + StatusHandler.log(new Status(IStatus.ERROR, TasksIndexCore.BUNDLE_ID, + "Unexpected failure within task list index", e)); //$NON-NLS-1$ + } finally { + try { + indexSearcher.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + } + } finally { + readLock.unlock(); + } + } + + private Query computeQuery(String patternString) { + String upperPatternString = patternString.toUpperCase(); + + boolean hasBooleanSpecifiers = upperPatternString.contains(" OR ") || upperPatternString.contains(" AND ") //$NON-NLS-1$ //$NON-NLS-2$ + || upperPatternString.contains(" NOT "); //$NON-NLS-1$ + + if (patternString.indexOf(':') == -1 && !hasBooleanSpecifiers && defaultField == IndexField.SUMMARY + && patternString.indexOf('"') == -1) { + return new PrefixQuery(new Term(defaultField.fieldName(), patternString)); + } + QueryParser qp = new QueryParser(Version.LUCENE_CURRENT, defaultField.fieldName(), new TaskAnalyzer()); + Query q; + try { + q = qp.parse(patternString); + } catch (ParseException e) { + return new PrefixQuery(new Term(defaultField.fieldName(), patternString)); + } + + // relax term clauses to be prefix clauses so that we get results close + // to what we're expecting + // from previous task list search + if (q instanceof BooleanQuery) { + BooleanQuery query = (BooleanQuery) q; + for (BooleanClause clause : query.getClauses()) { + if (clause.getQuery() instanceof TermQuery) { + TermQuery termQuery = (TermQuery) clause.getQuery(); + clause.setQuery(new PrefixQuery(termQuery.getTerm())); + } + if (!hasBooleanSpecifiers) { + clause.setOccur(Occur.MUST); + } + } + } else if (q instanceof TermQuery) { + return new PrefixQuery(((TermQuery) q).getTerm()); + } + return q; + } + + public void close() { + dataManager.removeListener(this); + taskList.removeChangeListener(this); + + maintainIndexJob.cancel(); + try { + maintainIndexJob.join(); + } catch (InterruptedException e) { + // ignore + } + + Lock writeLock = indexReaderLock.writeLock(); + writeLock.lock(); + try { + synchronized (this) { + if (indexReader != null) { + try { + indexReader.close(); + } catch (IOException e) { + e.printStackTrace(); + } + indexReader = null; + } + } + try { + directory.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } finally { + writeLock.unlock(); + } + } + + private IndexReader getIndexReader() { + try { + synchronized (this) { + if (indexReader == null) { + indexReader = IndexReader.open(directory, true); + lastResults = null; + } + return indexReader; + } + } catch (CorruptIndexException e) { + rebuildIndex = true; + if (maintainIndexJob != null) { + scheduleIndexMaintenance(MaintainIndexType.REINDEX); + } + } catch (FileNotFoundException e) { + rebuildIndex = true; + // expected if the index doesn't exist + } catch (IOException e) { + e.printStackTrace(); + } + return null; + } + + public void taskDataUpdated(TaskDataManagerEvent event) { + reindex(event.getTask(), event.getTaskData()); + } + + public void editsDiscarded(TaskDataManagerEvent event) { + reindex(event.getTask(), event.getTaskData()); + } + + public void containersChanged(Set<TaskContainerDelta> containers) { + for (TaskContainerDelta delta : containers) { + switch (delta.getKind()) { + case ADDED: + case REMOVED: + case CONTENT: + IRepositoryElement element = delta.getElement(); + if (element instanceof ITask) { + ITask task = (ITask) element; + if ("local".equals(((AbstractTask) task).getConnectorKind())) { //$NON-NLS-1$ + reindex(task, null); + } + } + } + } + } + + /** + * advanced usage: cause the given task to be reindexed using {@link MaintainIndexType#REINDEX reindex scheduling + * rule}. + * + * @param task + * the task + * @param taskData + * the task data, or nul if it's not available + */ + protected void reindex(ITask task, TaskData taskData) { + if (task == null) { + // this can happen when edits are discarded + return; + } + if (!taskIsIndexable(task, taskData)) { + return; + } + synchronized (reindexQueue) { + reindexQueue.put(task, taskData); + } + scheduleIndexMaintenance(MaintainIndexType.REINDEX); + } + + private void addIndexedAttributes(Document document, ITask task, TaskAttribute root) { + addIndexedAttribute(document, IndexField.TASK_KEY, task.getTaskKey()); + addIndexedAttribute(document, IndexField.REPOSITORY_URL, task.getRepositoryUrl()); + addIndexedAttribute(document, IndexField.SUMMARY, root.getMappedAttribute(TaskAttribute.SUMMARY)); + + for (TaskAttribute contentAttribute : computeContentAttributes(root)) { + addIndexedAttribute(document, IndexField.CONTENT, contentAttribute); + } + + addIndexedDateAttributes(document, task); + + List<TaskAttribute> commentAttributes = root.getTaskData() + .getAttributeMapper() + .getAttributesByType(root.getTaskData(), TaskAttribute.TYPE_COMMENT); + for (TaskAttribute commentAttribute : commentAttributes) { + TaskCommentMapper commentMapper = TaskCommentMapper.createFrom(commentAttribute); + String text = commentMapper.getText(); + if (text.length() != 0) { + addIndexedAttribute(document, IndexField.CONTENT, text); + } + IRepositoryPerson author = commentMapper.getAuthor(); + if (author != null) { + addIndexedAttribute(document, IndexField.PERSON, author.getPersonId()); + } + } + + List<TaskAttribute> personAttributes = root.getTaskData() + .getAttributeMapper() + .getAttributesByType(root.getTaskData(), TaskAttribute.TYPE_PERSON); + for (TaskAttribute personAttribute : personAttributes) { + addIndexedAttribute(document, IndexField.PERSON, personAttribute); + } + + for (IndexField field : IndexField.values()) { + if (field.getAttributeId() != null) { + addIndexedAttribute(document, field, root.getMappedAttribute(field.getAttributeId())); + } + } + } + + /** + * compute attributes that should be indexed as {@link IndexField#CONTENT} + */ + private Collection<TaskAttribute> computeContentAttributes(TaskAttribute root) { + Set<TaskAttribute> attributes = new LinkedHashSet<TaskAttribute>(); + + // add default content attributes + { + TaskAttribute attribute = root.getMappedAttribute(TaskAttribute.SUMMARY); + if (attribute != null) { + attributes.add(attribute); + } + attribute = root.getMappedAttribute(TaskAttribute.DESCRIPTION); + if (attribute != null) { + attributes.add(attribute); + } + // bugzilla + attribute = root.getAttribute("status_whiteboard"); //$NON-NLS-1$ + if (attribute != null) { + attributes.add(attribute); + } + } + + for (TaskAttribute attribute : root.getAttributes().values()) { + if (Boolean.parseBoolean(attribute.getMetaData().getValue(META_INDEXED_AS_CONTENT))) { + attributes.add(attribute); + } + } + + return attributes; + } + + private void addIndexedAttributes(Document document, ITask task) { + addIndexedAttribute(document, IndexField.TASK_KEY, task.getTaskKey()); + addIndexedAttribute(document, IndexField.REPOSITORY_URL, task.getRepositoryUrl()); + addIndexedAttribute(document, IndexField.SUMMARY, task.getSummary()); + addIndexedAttribute(document, IndexField.CONTENT, task.getSummary()); + addIndexedAttribute(document, IndexField.CONTENT, ((AbstractTask) task).getNotes()); + addIndexedDateAttributes(document, task); + } + + private void addIndexedDateAttributes(Document document, ITask task) { + addIndexedAttribute(document, IndexField.COMPLETION_DATE, task.getCompletionDate()); + addIndexedAttribute(document, IndexField.CREATION_DATE, task.getCreationDate()); + addIndexedAttribute(document, IndexField.DUE_DATE, task.getDueDate()); + addIndexedAttribute(document, IndexField.MODIFICATION_DATE, task.getModificationDate()); + } + + private void addIndexedAttribute(Document document, IndexField indexField, TaskAttribute attribute) { + if (attribute == null) { + return; + } + // if (indexField == IndexField.ASSIGNEE) { + // System.out.println(indexField + "=" + attribute.getValue()); + // } + List<String> values = attribute.getValues(); + for (String value : values) { + if (value.length() != 0) { + addIndexedAttribute(document, indexField, value); + } + } + } + + private void addIndexedAttribute(Document document, IndexField indexField, String value) { + if (value == null) { + return; + } + Field field = document.getField(indexField.fieldName()); + if (field == null) { + field = new Field(indexField.fieldName(), value, Store.YES, org.apache.lucene.document.Field.Index.ANALYZED); + document.add(field); + } else { + String existingValue = field.stringValue(); + if (indexField != IndexField.PERSON || !existingValue.contains(value)) { + field.setValue(existingValue + " " + value); //$NON-NLS-1$ + } + } + } + + private void addIndexedAttribute(Document document, IndexField indexField, Date date) { + if (date == null) { + return; + } + // FIXME: date tools converts dates to GMT, and we don't really want that. So + // move the date by the GMT offset if there is any + + String value = DateTools.dateToString(date, Resolution.HOUR); + Field field = document.getField(indexField.fieldName()); + if (field == null) { + field = new Field(indexField.fieldName(), value, Store.YES, org.apache.lucene.document.Field.Index.ANALYZED); + document.add(field); + } else { + field.setValue(value); + } + } + + private class MaintainIndexJob extends Job { + + public MaintainIndexJob() { + super(Messages.TaskListIndex_indexerJob); + setUser(false); + setSystem(false); // true? + setPriority(Job.LONG); + } + + @Override + public IStatus run(IProgressMonitor m) { + final int WORK_PER_SEGMENT = 1000; + SubMonitor monitor = SubMonitor.convert(m, 3 * WORK_PER_SEGMENT); + try { + try { + if (monitor.isCanceled()) { + return Status.CANCEL_STATUS; + } + if (!rebuildIndex) { + try { + IndexReader reader = IndexReader.open(directory, false); + reader.close(); + } catch (CorruptIndexException e) { + rebuildIndex = true; + } + } + + if (rebuildIndex) { + synchronized (reindexQueue) { + reindexQueue.clear(); + } + + SubMonitor reindexMonitor = monitor.newChild(WORK_PER_SEGMENT); + + final IndexWriter writer = new IndexWriter(directory, new TaskAnalyzer(), true, + IndexWriter.MaxFieldLength.UNLIMITED); + try { + + final List<ITask> allTasks = new ArrayList<ITask>(5000); + + taskList.run(new ITaskListRunnable() { + public void execute(IProgressMonitor monitor) throws CoreException { + for (ITask task : taskList.getAllTasks()) { + if (taskIsIndexable(task, null)) { + allTasks.add(task); + } + } + } + }, monitor.newChild(1)); + + int reindexErrorCount = 0; + + reindexMonitor.beginTask(Messages.TaskListIndex_task_rebuildingIndex, allTasks.size()); + for (ITask task : allTasks) { + if (!taskIsIndexable(task, null)) { + continue; + } + try { + TaskData taskData = dataManager.getTaskData(task); + add(writer, task, taskData); + + reindexMonitor.worked(1); + } catch (CoreException e) { + // an individual task data error should not prevent the index from updating + // but don't flood the log in the case of multiple errors + if (reindexErrorCount++ == 0) { + StatusHandler.log(e.getStatus()); + } + } catch (IOException e) { + throw e; + } + } + synchronized (TaskListIndex.this) { + rebuildIndex = false; + } + } finally { + writer.close(); + reindexMonitor.done(); + } + } else { + monitor.worked(WORK_PER_SEGMENT); + } + for (;;) { + + synchronized (reindexQueue) { + if (reindexQueue.isEmpty()) { + break; + } + } + + Map<ITask, TaskData> queue = new HashMap<ITask, TaskData>(); + + IndexReader reader = IndexReader.open(directory, false); + try { + synchronized (reindexQueue) { + queue.putAll(reindexQueue); + for (ITask task : queue.keySet()) { + reindexQueue.remove(task); + } + } + Iterator<Entry<ITask, TaskData>> it = queue.entrySet().iterator(); + while (it.hasNext()) { + Entry<ITask, TaskData> entry = it.next(); + + reader.deleteDocuments(new Term(IndexField.IDENTIFIER.fieldName(), entry.getKey() + .getHandleIdentifier())); + + } + } finally { + reader.close(); + } + monitor.worked(WORK_PER_SEGMENT); + + IndexWriter writer = new IndexWriter(directory, new TaskAnalyzer(), false, + IndexWriter.MaxFieldLength.UNLIMITED); + try { + for (Entry<ITask, TaskData> entry : queue.entrySet()) { + ITask task = entry.getKey(); + TaskData taskData = entry.getValue(); + + add(writer, task, taskData); + } + } finally { + writer.close(); + } + monitor.worked(WORK_PER_SEGMENT); + } + Lock writeLock = indexReaderLock.writeLock(); + writeLock.lock(); + try { + synchronized (TaskListIndex.this) { + if (indexReader != null) { + indexReader.close(); + indexReader = null; + } + } + } finally { + writeLock.unlock(); + } + } catch (CoreException e) { + throw e; + } catch (Throwable e) { + throw new CoreException(new Status(IStatus.ERROR, TasksIndexCore.BUNDLE_ID, + "Unexpected exception: " + e.getMessage(), e)); //$NON-NLS-1$ + } + } catch (CoreException e) { + MultiStatus logStatus = new MultiStatus(TasksIndexCore.BUNDLE_ID, 0, + "Failed to update task list index", e); //$NON-NLS-1$ + logStatus.add(e.getStatus()); + StatusHandler.log(logStatus); + } finally { + monitor.done(); + } + return Status.OK_STATUS; + } + + /** + * @param writer + * @param task + * the task + * @param taskData + * may be null for local tasks + * @throws CorruptIndexException + * @throws IOException + */ + private void add(IndexWriter writer, ITask task, TaskData taskData) throws CorruptIndexException, IOException { + if (!taskIsIndexable(task, taskData)) { + return; + } + + Document document = new Document(); + + document.add(new Field(IndexField.IDENTIFIER.fieldName(), task.getHandleIdentifier(), Store.YES, + org.apache.lucene.document.Field.Index.ANALYZED)); + if (taskData == null) { + if ("local".equals(((AbstractTask) task).getConnectorKind())) { //$NON-NLS-1$ + addIndexedAttributes(document, task); + } else { + return; + } + } else { + addIndexedAttributes(document, task, taskData.getRoot()); + } + writer.addDocument(document); + } + + } + + /** + * Computes a query element for a field that must lie in a specified date range. + * + * @param field + * the field + * @param lowerBoundInclusive + * the date lower bound that the field value must match, inclusive + * @param upperBoundInclusive + * the date upper bound that the field value must match, inclusive + * @return + */ + public String computeQueryFieldDateRange(IndexField field, Date lowerBoundInclusive, Date upperBoundInclusive) { + return field.fieldName() + + ":[" + DateTools.dateToString(lowerBoundInclusive, Resolution.DAY) + " TO " + DateTools.dateToString(upperBoundInclusive, Resolution.DAY) + "]"; //$NON-NLS-1$//$NON-NLS-2$//$NON-NLS-3$ + } + + /** + * Indicates if the given task is indexable. The default implementation returns true, subclasses may override to + * filter some tasks from the task list. This method may be called more than once per task, with some calls omitting + * the task data. In this way implementations can avoid loading task data if the decision to filter tasks can be + * based on the ITask alone. Implementations that must read the task data in order to determine eligibility for + * indexing should return true for tasks where the provided task data is null. + * + * @param task + * the task + * @param taskData + * the task data, or null if there is no task data + * @return true if the given task should be indexed, otherwise false. + */ + protected boolean taskIsIndexable(ITask task, TaskData taskData) { + return true; + } + + /** + * Escapes special characters in the given literal value so that they are not interpreted as special characters in a + * query. + * + * @param value + * the value to escape + * @return a representation of the value with characters escaped + */ + public String escapeFieldValue(String value) { + // see http://lucene.apache.org/java/2_9_1/queryparsersyntax.html#Escaping%20Special%20Characters + String escaped = value.replaceAll("([\\+\\-\\!\\(\\)\\{\\}\\[\\]^\"~\\*\\?:\\\\]|&&|\\|\\|)", "\\\\$1"); //$NON-NLS-1$ //$NON-NLS-2$ + return escaped; + } +} diff --git a/org.eclipse.mylyn.tasks.index.core/src/org/eclipse/mylyn/internal/tasks/index/core/TasksIndexCore.java b/org.eclipse.mylyn.tasks.index.core/src/org/eclipse/mylyn/internal/tasks/index/core/TasksIndexCore.java new file mode 100644 index 000000000..2558063b4 --- /dev/null +++ b/org.eclipse.mylyn.tasks.index.core/src/org/eclipse/mylyn/internal/tasks/index/core/TasksIndexCore.java @@ -0,0 +1,19 @@ +/******************************************************************************* + * Copyright (c) 2011 Tasktop Technologies. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * Contributors: + * Tasktop Technologies - initial API and implementation + *******************************************************************************/ +package org.eclipse.mylyn.internal.tasks.index.core; + +/** + * @author David Green + */ +public class TasksIndexCore { + public static final String BUNDLE_ID = "org.eclipse.mylyn.tasks.index.core"; //$NON-NLS-1$ + +} diff --git a/org.eclipse.mylyn.tasks.index.core/src/org/eclipse/mylyn/internal/tasks/index/core/messages.properties b/org.eclipse.mylyn.tasks.index.core/src/org/eclipse/mylyn/internal/tasks/index/core/messages.properties new file mode 100644 index 000000000..5354a0937 --- /dev/null +++ b/org.eclipse.mylyn.tasks.index.core/src/org/eclipse/mylyn/internal/tasks/index/core/messages.properties @@ -0,0 +1,12 @@ +############################################################################### +# Copyright (c) 2011 Tasktop Technologies and others. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +# +# Contributors: +# Tasktop Technologies - initial API and implementation +############################################################################### +TaskListIndex_indexerJob=Task List Indexer +TaskListIndex_task_rebuildingIndex=Rebuilding Task List Index |