path: root/org.eclipse.mylyn.tasks.index.core/src/org/eclipse/mylyn/internal/tasks/index/core/TaskListIndex.java

/*******************************************************************************
 * Copyright (c) 2011 Tasktop Technologies.
 * All rights reserved. This program and the accompanying materials
 * are made available under the terms of the Eclipse Public License v1.0
 * which accompanies this distribution, and is available at
 * http://www.eclipse.org/legal/epl-v10.html
 *
 * Contributors:
 *     Tasktop Technologies - initial API and implementation
 *******************************************************************************/

package org.eclipse.mylyn.internal.tasks.index.core;

import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.logging.Logger;

import org.apache.lucene.document.DateTools;
import org.apache.lucene.document.DateTools.Resolution;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.Field.Store;
import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.Term;
import org.apache.lucene.queryParser.ParseException;
import org.apache.lucene.queryParser.QueryParser;
import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.BooleanClause.Occur;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.PrefixQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.NIOFSDirectory;
import org.apache.lucene.util.Version;
import org.eclipse.core.runtime.CoreException;
import org.eclipse.core.runtime.IProgressMonitor;
import org.eclipse.core.runtime.IStatus;
import org.eclipse.core.runtime.MultiStatus;
import org.eclipse.core.runtime.NullProgressMonitor;
import org.eclipse.core.runtime.Platform;
import org.eclipse.core.runtime.Status;
import org.eclipse.core.runtime.SubMonitor;
import org.eclipse.core.runtime.jobs.Job;
import org.eclipse.mylyn.commons.core.StatusHandler;
import org.eclipse.mylyn.internal.tasks.core.AbstractTask;
import org.eclipse.mylyn.internal.tasks.core.ITaskListChangeListener;
import org.eclipse.mylyn.internal.tasks.core.ITaskListRunnable;
import org.eclipse.mylyn.internal.tasks.core.TaskContainerDelta;
import org.eclipse.mylyn.internal.tasks.core.TaskList;
import org.eclipse.mylyn.internal.tasks.core.data.ITaskDataManagerListener;
import org.eclipse.mylyn.internal.tasks.core.data.TaskDataManager;
import org.eclipse.mylyn.internal.tasks.core.data.TaskDataManagerEvent;
import org.eclipse.mylyn.tasks.core.IRepositoryElement;
import org.eclipse.mylyn.tasks.core.IRepositoryPerson;
import org.eclipse.mylyn.tasks.core.ITask;
import org.eclipse.mylyn.tasks.core.data.TaskAttribute;
import org.eclipse.mylyn.tasks.core.data.TaskCommentMapper;
import org.eclipse.mylyn.tasks.core.data.TaskData;

/**
 * An index on a task list.
 * 
 * @author David Green
 */
public class TaskListIndex implements ITaskDataManagerListener, ITaskListChangeListener {

	public abstract static class TaskCollector {

		public abstract void collect(ITask task);

	}

	private static final Object COMMAND_RESET_INDEX = "index:reset"; //$NON-NLS-1$

	/**
	 * Task attribute meta-data key that should be set to "true" to have attribute value indexed as part of the task
	 * {@link IndexField#CONTENT}. Provides a way for connectors to specify non-standard attributes as indexable. By
	 * default, {@link TaskAttribute#SUMMARY summary} and {@link TaskAttribute#DESCRIPTION description} are indexed.
	 */
	public static final String META_INDEXED_AS_CONTENT = "index-content"; //$NON-NLS-1$

	public static enum IndexField {
		IDENTIFIER(false, null, false), //
		TASK_KEY(false, null, false), //
		REPOSITORY_URL(false, null, false), //
		SUMMARY(true, null, false), //
		CONTENT(true, null, false), //
		ASSIGNEE(true, TaskAttribute.USER_ASSIGNED, false), //
		REPORTER(true, TaskAttribute.USER_REPORTER, false), //
		PERSON(true, null, false), //
		COMPONENT(true, TaskAttribute.COMPONENT, false), //
		COMPLETION_DATE(true, null, true), //
		CREATION_DATE(true, null, true), //
		DUE_DATE(true, null, true), //
		MODIFICATION_DATE(true, null, true), //
		DESCRIPTION(true, TaskAttribute.DESCRIPTION, false), //
		KEYWORDS(true, TaskAttribute.KEYWORDS, false), //
		PRODUCT(true, TaskAttribute.PRODUCT, false), //
		RESOLUTION(true, TaskAttribute.RESOLUTION, false), //
		SEVERITY(true, TaskAttribute.SEVERITY, false), //
		STATUS(true, TaskAttribute.STATUS, false);

		private final boolean userVisible;

		private final String attributeId;

		private final boolean dateTime;

		private IndexField(boolean userVisible, String attributeId, boolean dateTime) {
			this.userVisible = userVisible;
			this.attributeId = attributeId;
			this.dateTime = dateTime;
		}

		public String fieldName() {
			return name().toLowerCase();
		}

		/**
		 * get the task attribute id, or null if this field has special handling
		 */
		public String getAttributeId() {
			return attributeId;
		}

		/**
		 * indicate if the field should be exposed in the UI
		 */
		public boolean isUserVisible() {
			return userVisible;
		}

		/**
		 * indicate if the field is a date/time field
		 */
		public boolean isDateTime() {
			return dateTime;
		}

		public static IndexField fromFieldName(String fieldName) {
			try {
				return IndexField.valueOf(fieldName.toUpperCase());
			} catch (IllegalArgumentException e) {
				return null;
			}
		}

		public boolean isPersonField() {
			return this == PERSON || this == REPORTER || this == ASSIGNEE;
		}
	}

	private static enum MaintainIndexType {
		STARTUP, REINDEX
	}

	private Directory directory;

	private MaintainIndexJob maintainIndexJob;

	private final Map<ITask, TaskData> reindexQueue = new HashMap<ITask, TaskData>();

	private IndexReader indexReader;

	private boolean rebuildIndex = false;

	private String lastPatternString;

	private Set<String> lastResults;

	private IndexField defaultField = IndexField.SUMMARY;

	private final TaskList taskList;

	private final TaskDataManager dataManager;

	private long startupDelay = 6000L;

	private long reindexDelay = 3000L;

	private int maxMatchSearchHits = 1500;

	private final ReadWriteLock indexReaderLock = new ReentrantReadWriteLock(true);

	private TaskListIndex(TaskList taskList, TaskDataManager dataManager) {
		if (taskList == null) {
			throw new IllegalArgumentException();
		}
		if (dataManager == null) {
			throw new IllegalArgumentException();
		}
		this.taskList = taskList;
		this.dataManager = dataManager;
	}

	public TaskListIndex(TaskList taskList, TaskDataManager dataManager, File indexLocation) {
		this(taskList, dataManager, indexLocation, 6000L);
	}

	public TaskListIndex(TaskList taskList, TaskDataManager dataManager, File indexLocation, long startupDelay) {
		this(taskList, dataManager);
		if (startupDelay < 0L || startupDelay > (1000L * 60)) {
			throw new IllegalArgumentException();
		}
		if (indexLocation == null) {
			throw new IllegalArgumentException();
		}
		this.startupDelay = startupDelay;
		if (!indexLocation.exists()) {
			rebuildIndex = true;
			if (!indexLocation.mkdirs()) {
				StatusHandler.log(new Status(IStatus.ERROR, TasksIndexCore.BUNDLE_ID,
						"Cannot create task list index folder: " + indexLocation)); //$NON-NLS-1$
			}
		}
		if (indexLocation.exists() && indexLocation.isDirectory()) {
			try {
				Logger.getLogger(TaskListIndex.class.getName()).fine("task list index: " + indexLocation); //$NON-NLS-1$

				directory = new NIOFSDirectory(indexLocation);
			} catch (IOException e) {
				StatusHandler.log(new Status(IStatus.ERROR, TasksIndexCore.BUNDLE_ID,
						"Cannot create task list index", e)); //$NON-NLS-1$
			}
		}
		initialize();
	}

	public TaskListIndex(TaskList taskList, TaskDataManager dataManager, Directory directory) {
		this(taskList, dataManager);
		this.directory = directory;
		initialize();
	}

	public long getReindexDelay() {
		return reindexDelay;
	}

	public void setReindexDelay(long reindexDelay) {
		this.reindexDelay = reindexDelay;
	}

	public IndexField getDefaultField() {
		return defaultField;
	}

	public void setDefaultField(IndexField defaultField) {
		this.defaultField = defaultField;
		synchronized (this) {
			lastResults = null;
		}
	}

	public int getMaxMatchSearchHits() {
		return maxMatchSearchHits;
	}

	public void setMaxMatchSearchHits(int maxMatchSearchHits) {
		this.maxMatchSearchHits = maxMatchSearchHits;
	}

	private void initialize() {
		if (!rebuildIndex) {
			IndexReader indexReader = null;
			try {
				indexReader = getIndexReader();
			} catch (Exception e) {
				// ignore, this can happen if the index is corrupt
			}
			if (indexReader == null) {
				rebuildIndex = true;
			}
		}
		maintainIndexJob = new MaintainIndexJob();
		dataManager.addListener(this);
		taskList.addChangeListener(this);

		scheduleIndexMaintenance(MaintainIndexType.STARTUP);
	}

	private void scheduleIndexMaintenance(MaintainIndexType type) {
		long delay = 0L;
		switch (type) {
		case STARTUP:
			delay = startupDelay;
			break;
		case REINDEX:
			delay = reindexDelay;
		}

		if (delay == 0L) {
			// primarily for testing purposes

			maintainIndexJob.cancel();
			try {
				maintainIndexJob.join();
			} catch (InterruptedException e) {
				// ignore
			}
			maintainIndexJob.run(new NullProgressMonitor());
		} else {
			maintainIndexJob.schedule(delay);
		}
	}

	public boolean matches(ITask task, String patternString) {
		if (patternString.equals(COMMAND_RESET_INDEX)) {
			reindex();
		}
		Lock readLock = indexReaderLock.readLock();
		readLock.lock();
		try {

			IndexReader indexReader = getIndexReader();
			if (indexReader != null) {
				Set<String> hits;

				if (lastResults == null || (lastPatternString == null || !lastPatternString.equals(patternString))) {
					this.lastPatternString = patternString;

					long startTime = System.currentTimeMillis();

					hits = new HashSet<String>();

					IndexSearcher indexSearcher = new IndexSearcher(indexReader);
					try {
						Query query = computeQuery(patternString);
						TopDocs results = indexSearcher.search(query, maxMatchSearchHits);
						for (ScoreDoc scoreDoc : results.scoreDocs) {
							Document document = indexReader.document(scoreDoc.doc);
							hits.add(document.get(IndexField.IDENTIFIER.fieldName()));
						}
					} catch (IOException e) {
						StatusHandler.log(new Status(IStatus.ERROR, TasksIndexCore.BUNDLE_ID,
								"Unexpected failure within task list index", e)); //$NON-NLS-1$
					} finally {
						try {
							indexSearcher.close();
						} catch (IOException e) {
							// ignore
						}
					}

					Logger.getLogger(TaskListIndex.class.getName()).fine(
							"New query in " + (System.currentTimeMillis() - startTime) + "ms"); //$NON-NLS-1$ //$NON-NLS-2$
				} else {
					hits = lastResults;
				}
				synchronized (this) {
					if (this.indexReader == indexReader) {
						this.lastPatternString = patternString;
						this.lastResults = hits;
					}
				}
				String taskIdentifier = task.getHandleIdentifier();
				return hits != null && hits.contains(taskIdentifier);
			}

		} finally {
			readLock.unlock();
		}
		return false;
	}

	public void reindex() {
		rebuildIndex = true;
		scheduleIndexMaintenance(MaintainIndexType.REINDEX);
	}

	/**
	 * call to wait until index maintenance has completed
	 * 
	 * @throws InterruptedException
	 */
	public void waitUntilIdle() throws InterruptedException {
		if (!Platform.isRunning() && reindexDelay != 0L) {
			// job join() behaviour is not the same when platform is not running
			Logger.getLogger(TaskListIndex.class.getName()).warning(
					"Index job joining may not work properly when Eclipse platform is not running"); //$NON-NLS-1$
		}
		maintainIndexJob.join();
	}

	public void find(String patternString, TaskCollector collector, int resultsLimit) {

		Lock readLock = indexReaderLock.readLock();
		readLock.lock();
		try {
			IndexReader indexReader = getIndexReader();
			if (indexReader != null) {
				IndexSearcher indexSearcher = new IndexSearcher(indexReader);
				try {
					Query query = computeQuery(patternString);
					TopDocs results = indexSearcher.search(query, resultsLimit);
					for (ScoreDoc scoreDoc : results.scoreDocs) {
						Document document = indexReader.document(scoreDoc.doc);
						String taskIdentifier = document.get(IndexField.IDENTIFIER.fieldName());
						AbstractTask task = taskList.getTask(taskIdentifier);
						if (task != null) {
							collector.collect(task);
						}
					}
				} catch (IOException e) {
					StatusHandler.log(new Status(IStatus.ERROR, TasksIndexCore.BUNDLE_ID,
							"Unexpected failure within task list index", e)); //$NON-NLS-1$
				} finally {
					try {
						indexSearcher.close();
					} catch (IOException e) {
						e.printStackTrace();
					}
				}
			}
		} finally {
			readLock.unlock();
		}
	}

	private Query computeQuery(String patternString) {
		String upperPatternString = patternString.toUpperCase();

		boolean hasBooleanSpecifiers = upperPatternString.contains(" OR ") || upperPatternString.contains(" AND ") //$NON-NLS-1$ //$NON-NLS-2$
				|| upperPatternString.contains(" NOT "); //$NON-NLS-1$

		if (patternString.indexOf(':') == -1 && !hasBooleanSpecifiers && defaultField == IndexField.SUMMARY
				&& patternString.indexOf('"') == -1) {
			return new PrefixQuery(new Term(defaultField.fieldName(), patternString));
		}
		QueryParser qp = new QueryParser(Version.LUCENE_CURRENT, defaultField.fieldName(), new TaskAnalyzer());
		Query q;
		try {
			q = qp.parse(patternString);
		} catch (ParseException e) {
			return new PrefixQuery(new Term(defaultField.fieldName(), patternString));
		}

		// relax term clauses to be prefix clauses so that we get results close
		// to what we're expecting
		// from previous task list search
		if (q instanceof BooleanQuery) {
			BooleanQuery query = (BooleanQuery) q;
			for (BooleanClause clause : query.getClauses()) {
				if (clause.getQuery() instanceof TermQuery) {
					TermQuery termQuery = (TermQuery) clause.getQuery();
					clause.setQuery(new PrefixQuery(termQuery.getTerm()));
				}
				if (!hasBooleanSpecifiers) {
					clause.setOccur(Occur.MUST);
				}
			}
		} else if (q instanceof TermQuery) {
			return new PrefixQuery(((TermQuery) q).getTerm());
		}
		return q;
	}

	public void close() {
		dataManager.removeListener(this);
		taskList.removeChangeListener(this);

		maintainIndexJob.cancel();
		try {
			maintainIndexJob.join();
		} catch (InterruptedException e) {
			// ignore
		}

		Lock writeLock = indexReaderLock.writeLock();
		writeLock.lock();
		try {
			synchronized (this) {
				if (indexReader != null) {
					try {
						indexReader.close();
					} catch (IOException e) {
						e.printStackTrace();
					}
					indexReader = null;
				}
			}
			try {
				directory.close();
			} catch (IOException e) {
				e.printStackTrace();
			}
		} finally {
			writeLock.unlock();
		}
	}

	private IndexReader getIndexReader() {
		try {
			synchronized (this) {
				if (indexReader == null) {
					indexReader = IndexReader.open(directory, true);
					lastResults = null;
				}
				return indexReader;
			}
		} catch (CorruptIndexException e) {
			rebuildIndex = true;
			if (maintainIndexJob != null) {
				scheduleIndexMaintenance(MaintainIndexType.REINDEX);
			}
		} catch (FileNotFoundException e) {
			rebuildIndex = true;
			// expected if the index doesn't exist
		} catch (IOException e) {
			e.printStackTrace();
		}
		return null;
	}

	public void taskDataUpdated(TaskDataManagerEvent event) {
		reindex(event.getTask(), event.getTaskData());
	}

	public void editsDiscarded(TaskDataManagerEvent event) {
		reindex(event.getTask(), event.getTaskData());
	}

	public void containersChanged(Set<TaskContainerDelta> containers) {
		for (TaskContainerDelta delta : containers) {
			switch (delta.getKind()) {
			case ADDED:
			case REMOVED:
			case CONTENT:
				IRepositoryElement element = delta.getElement();
				if (element instanceof ITask) {
					ITask task = (ITask) element;
					if ("local".equals(((AbstractTask) task).getConnectorKind())) { //$NON-NLS-1$
						reindex(task, null);
					}
				}
			}
		}
	}

	/**
	 * advanced usage: cause the given task to be reindexed using {@link MaintainIndexType#REINDEX reindex scheduling
	 * rule}.
	 * 
	 * @param task
	 *            the task
	 * @param taskData
	 *            the task data, or nul if it's not available
	 */
	protected void reindex(ITask task, TaskData taskData) {
		if (task == null) {
			// this can happen when edits are discarded
			return;
		}
		if (!taskIsIndexable(task, taskData)) {
			return;
		}
		synchronized (reindexQueue) {
			reindexQueue.put(task, taskData);
		}
		scheduleIndexMaintenance(MaintainIndexType.REINDEX);
	}

	private void addIndexedAttributes(Document document, ITask task, TaskAttribute root) {
		addIndexedAttribute(document, IndexField.TASK_KEY, task.getTaskKey());
		addIndexedAttribute(document, IndexField.REPOSITORY_URL, task.getRepositoryUrl());
		addIndexedAttribute(document, IndexField.SUMMARY, root.getMappedAttribute(TaskAttribute.SUMMARY));

		for (TaskAttribute contentAttribute : computeContentAttributes(root)) {
			addIndexedAttribute(document, IndexField.CONTENT, contentAttribute);
		}

		addIndexedDateAttributes(document, task);

		List<TaskAttribute> commentAttributes = root.getTaskData()
				.getAttributeMapper()
				.getAttributesByType(root.getTaskData(), TaskAttribute.TYPE_COMMENT);
		for (TaskAttribute commentAttribute : commentAttributes) {
			TaskCommentMapper commentMapper = TaskCommentMapper.createFrom(commentAttribute);
			String text = commentMapper.getText();
			if (text.length() != 0) {
				addIndexedAttribute(document, IndexField.CONTENT, text);
			}
			IRepositoryPerson author = commentMapper.getAuthor();
			if (author != null) {
				addIndexedAttribute(document, IndexField.PERSON, author.getPersonId());
			}
		}

		List<TaskAttribute> personAttributes = root.getTaskData()
				.getAttributeMapper()
				.getAttributesByType(root.getTaskData(), TaskAttribute.TYPE_PERSON);
		for (TaskAttribute personAttribute : personAttributes) {
			addIndexedAttribute(document, IndexField.PERSON, personAttribute);
		}

		for (IndexField field : IndexField.values()) {
			if (field.getAttributeId() != null) {
				addIndexedAttribute(document, field, root.getMappedAttribute(field.getAttributeId()));
			}
		}
	}

	/**
	 * compute attributes that should be indexed as {@link IndexField#CONTENT}
	 */
	private Collection<TaskAttribute> computeContentAttributes(TaskAttribute root) {
		Set<TaskAttribute> attributes = new LinkedHashSet<TaskAttribute>();

		// add default content attributes
		{
			TaskAttribute attribute = root.getMappedAttribute(TaskAttribute.SUMMARY);
			if (attribute != null) {
				attributes.add(attribute);
			}
			attribute = root.getMappedAttribute(TaskAttribute.DESCRIPTION);
			if (attribute != null) {
				attributes.add(attribute);
			}
			// bugzilla
			attribute = root.getAttribute("status_whiteboard"); //$NON-NLS-1$
			if (attribute != null) {
				attributes.add(attribute);
			}
		}

		for (TaskAttribute attribute : root.getAttributes().values()) {
			if (Boolean.parseBoolean(attribute.getMetaData().getValue(META_INDEXED_AS_CONTENT))) {
				attributes.add(attribute);
			}
		}

		return attributes;
	}

	private void addIndexedAttributes(Document document, ITask task) {
		addIndexedAttribute(document, IndexField.TASK_KEY, task.getTaskKey());
		addIndexedAttribute(document, IndexField.REPOSITORY_URL, task.getRepositoryUrl());
		addIndexedAttribute(document, IndexField.SUMMARY, task.getSummary());
		addIndexedAttribute(document, IndexField.CONTENT, task.getSummary());
		addIndexedAttribute(document, IndexField.CONTENT, ((AbstractTask) task).getNotes());
		addIndexedDateAttributes(document, task);
	}

	private void addIndexedDateAttributes(Document document, ITask task) {
		addIndexedAttribute(document, IndexField.COMPLETION_DATE, task.getCompletionDate());
		addIndexedAttribute(document, IndexField.CREATION_DATE, task.getCreationDate());
		addIndexedAttribute(document, IndexField.DUE_DATE, task.getDueDate());
		addIndexedAttribute(document, IndexField.MODIFICATION_DATE, task.getModificationDate());
	}

	private void addIndexedAttribute(Document document, IndexField indexField, TaskAttribute attribute) {
		if (attribute == null) {
			return;
		}
		// if (indexField == IndexField.ASSIGNEE) {
		// System.out.println(indexField + "=" + attribute.getValue());
		// }
		List<String> values = attribute.getValues();
		for (String value : values) {
			if (value.length() != 0) {
				addIndexedAttribute(document, indexField, value);
			}
		}
	}

	private void addIndexedAttribute(Document document, IndexField indexField, String value) {
		if (value == null) {
			return;
		}
		Field field = document.getField(indexField.fieldName());
		if (field == null) {
			field = new Field(indexField.fieldName(), value, Store.YES, org.apache.lucene.document.Field.Index.ANALYZED);
			document.add(field);
		} else {
			String existingValue = field.stringValue();
			if (indexField != IndexField.PERSON || !existingValue.contains(value)) {
				field.setValue(existingValue + " " + value); //$NON-NLS-1$
			}
		}
	}

	private void addIndexedAttribute(Document document, IndexField indexField, Date date) {
		if (date == null) {
			return;
		}
		// FIXME: date tools converts dates to GMT, and we don't really want that.  So
		// move the date by the GMT offset if there is any

		String value = DateTools.dateToString(date, Resolution.HOUR);
		Field field = document.getField(indexField.fieldName());
		if (field == null) {
			field = new Field(indexField.fieldName(), value, Store.YES, org.apache.lucene.document.Field.Index.ANALYZED);
			document.add(field);
		} else {
			field.setValue(value);
		}
	}

	private class MaintainIndexJob extends Job {

		public MaintainIndexJob() {
			super(Messages.TaskListIndex_indexerJob);
			setUser(false);
			setSystem(false); // true?
			setPriority(Job.LONG);
		}

		@Override
		public IStatus run(IProgressMonitor m) {
			final int WORK_PER_SEGMENT = 1000;
			SubMonitor monitor = SubMonitor.convert(m, 3 * WORK_PER_SEGMENT);
			try {
				try {
					if (monitor.isCanceled()) {
						return Status.CANCEL_STATUS;
					}
					if (!rebuildIndex) {
						try {
							IndexReader reader = IndexReader.open(directory, false);
							reader.close();
						} catch (CorruptIndexException e) {
							rebuildIndex = true;
						}
					}

					if (rebuildIndex) {
						synchronized (reindexQueue) {
							reindexQueue.clear();
						}

						SubMonitor reindexMonitor = monitor.newChild(WORK_PER_SEGMENT);

						final IndexWriter writer = new IndexWriter(directory, new TaskAnalyzer(), true,
								IndexWriter.MaxFieldLength.UNLIMITED);
						try {

							final List<ITask> allTasks = new ArrayList<ITask>(5000);

							taskList.run(new ITaskListRunnable() {
								public void execute(IProgressMonitor monitor) throws CoreException {
									for (ITask task : taskList.getAllTasks()) {
										if (taskIsIndexable(task, null)) {
											allTasks.add(task);
										}
									}
								}
							}, monitor.newChild(1));

							int reindexErrorCount = 0;

							reindexMonitor.beginTask(Messages.TaskListIndex_task_rebuildingIndex, allTasks.size());
							for (ITask task : allTasks) {
								if (!taskIsIndexable(task, null)) {
									continue;
								}
								try {
									TaskData taskData = dataManager.getTaskData(task);
									add(writer, task, taskData);

									reindexMonitor.worked(1);
								} catch (CoreException e) {
									// an individual task data error should not prevent the index from updating
									// but don't flood the log in the case of multiple errors
									if (reindexErrorCount++ == 0) {
										StatusHandler.log(e.getStatus());
									}
								} catch (IOException e) {
									throw e;
								}
							}
							synchronized (TaskListIndex.this) {
								rebuildIndex = false;
							}
						} finally {
							writer.close();
							reindexMonitor.done();
						}
					} else {
						monitor.worked(WORK_PER_SEGMENT);
					}
					for (;;) {

						synchronized (reindexQueue) {
							if (reindexQueue.isEmpty()) {
								break;
							}
						}

						Map<ITask, TaskData> queue = new HashMap<ITask, TaskData>();

						IndexReader reader = IndexReader.open(directory, false);
						try {
							synchronized (reindexQueue) {
								queue.putAll(reindexQueue);
								for (ITask task : queue.keySet()) {
									reindexQueue.remove(task);
								}
							}
							Iterator<Entry<ITask, TaskData>> it = queue.entrySet().iterator();
							while (it.hasNext()) {
								Entry<ITask, TaskData> entry = it.next();

								reader.deleteDocuments(new Term(IndexField.IDENTIFIER.fieldName(), entry.getKey()
										.getHandleIdentifier()));

							}
						} finally {
							reader.close();
						}
						monitor.worked(WORK_PER_SEGMENT);

						IndexWriter writer = new IndexWriter(directory, new TaskAnalyzer(), false,
								IndexWriter.MaxFieldLength.UNLIMITED);
						try {
							for (Entry<ITask, TaskData> entry : queue.entrySet()) {
								ITask task = entry.getKey();
								TaskData taskData = entry.getValue();

								add(writer, task, taskData);
							}
						} finally {
							writer.close();
						}
						monitor.worked(WORK_PER_SEGMENT);
					}
					Lock writeLock = indexReaderLock.writeLock();
					writeLock.lock();
					try {
						synchronized (TaskListIndex.this) {
							if (indexReader != null) {
								indexReader.close();
								indexReader = null;
							}
						}
					} finally {
						writeLock.unlock();
					}
				} catch (CoreException e) {
					throw e;
				} catch (Throwable e) {
					throw new CoreException(new Status(IStatus.ERROR, TasksIndexCore.BUNDLE_ID,
							"Unexpected exception: " + e.getMessage(), e)); //$NON-NLS-1$
				}
			} catch (CoreException e) {
				MultiStatus logStatus = new MultiStatus(TasksIndexCore.BUNDLE_ID, 0,
						"Failed to update task list index", e); //$NON-NLS-1$
				logStatus.add(e.getStatus());
				StatusHandler.log(logStatus);
			} finally {
				monitor.done();
			}
			return Status.OK_STATUS;
		}

		/**
		 * @param writer
		 * @param task
		 *            the task
		 * @param taskData
		 *            may be null for local tasks
		 * @throws CorruptIndexException
		 * @throws IOException
		 */
		private void add(IndexWriter writer, ITask task, TaskData taskData) throws CorruptIndexException, IOException {
			if (!taskIsIndexable(task, taskData)) {
				return;
			}

			Document document = new Document();

			document.add(new Field(IndexField.IDENTIFIER.fieldName(), task.getHandleIdentifier(), Store.YES,
					org.apache.lucene.document.Field.Index.ANALYZED));
			if (taskData == null) {
				if ("local".equals(((AbstractTask) task).getConnectorKind())) { //$NON-NLS-1$
					addIndexedAttributes(document, task);
				} else {
					return;
				}
			} else {
				addIndexedAttributes(document, task, taskData.getRoot());
			}
			writer.addDocument(document);
		}

	}

	/**
	 * Computes a query element for a field that must lie in a specified date range.
	 * 
	 * @param field
	 *            the field
	 * @param lowerBoundInclusive
	 *            the date lower bound that the field value must match, inclusive
	 * @param upperBoundInclusive
	 *            the date upper bound that the field value must match, inclusive
	 * @return
	 */
	public String computeQueryFieldDateRange(IndexField field, Date lowerBoundInclusive, Date upperBoundInclusive) {
		return field.fieldName()
				+ ":[" + DateTools.dateToString(lowerBoundInclusive, Resolution.DAY) + " TO " + DateTools.dateToString(upperBoundInclusive, Resolution.DAY) + "]"; //$NON-NLS-1$//$NON-NLS-2$//$NON-NLS-3$
	}

	/**
	 * Indicates if the given task is indexable. The default implementation returns true, subclasses may override to
	 * filter some tasks from the task list. This method may be called more than once per task, with some calls omitting
	 * the task data. In this way implementations can avoid loading task data if the decision to filter tasks can be
	 * based on the ITask alone. Implementations that must read the task data in order to determine eligibility for
	 * indexing should return true for tasks where the provided task data is null.
	 * 
	 * @param task
	 *            the task
	 * @param taskData
	 *            the task data, or null if there is no task data
	 * @return true if the given task should be indexed, otherwise false.
	 */
	protected boolean taskIsIndexable(ITask task, TaskData taskData) {
		return true;
	}

	/**
	 * Escapes special characters in the given literal value so that they are not interpreted as special characters in a
	 * query.
	 * 
	 * @param value
	 *            the value to escape
	 * @return a representation of the value with characters escaped
	 */
	public String escapeFieldValue(String value) {
		// see http://lucene.apache.org/java/2_9_1/queryparsersyntax.html#Escaping%20Special%20Characters
		String escaped = value.replaceAll("([\\+\\-\\!\\(\\)\\{\\}\\[\\]^\"~\\*\\?:\\\\]|&&|\\|\\|)", "\\\\$1"); //$NON-NLS-1$ //$NON-NLS-2$
		return escaped;
	}
}