Bug 525267 - Update platform.ua to Lucene 7I20171114-2000

This migrates platform code to be compatible with Lucene 7. Added tests
to support compatibility scenarios.

All search tests succeed.

Change-Id: Ied949b34b25fac9ef8a6f381c1c095f80bee05e4
Signed-off-by: Sopot Cela <scela@redhat.com>

21 files changed, 68 insertions, 26 deletions

diff --git a/org.eclipse.help.base/META-INF/MANIFEST.MF b/org.eclipse.help.base/META-INF/MANIFEST.MF
index d79de7560..9b1806173 100644
--- a/org.eclipse.help.base/META-INF/MANIFEST.MF
+++ b/org.eclipse.help.base/META-INF/MANIFEST.MF
@@ -44,10 +44,9 @@ Require-Bundle: org.eclipse.ant.core;bundle-version="[3.2.200,4.0.0)";resolution
  org.eclipse.help;bundle-version="[3.5.0,4.0.0)";visibility:=reexport,
  org.eclipse.core.expressions;bundle-version="[3.4.200,4.0.0)",
  org.eclipse.core.net;bundle-version="[1.2.200,2.0.0]",
- org.apache.lucene.analyzers-common;bundle-version="[6.1.0,7.0.0)",
- org.apache.lucene.core;bundle-version="[6.1.0,7.0.0)",
- org.apache.lucene.misc;bundle-version="[6.1.0,7.0.0)",
- org.apache.lucene.analyzers-smartcn;bundle-version="[6.1.0,7.0.0)"
+ org.apache.lucene.analyzers-common;bundle-version="[7.0.0,8.0.0)",
+ org.apache.lucene.core;bundle-version="[7.0.0,8.0.0)",
+ org.apache.lucene.analyzers-smartcn;bundle-version="[7.0.0,8.0.0)"
 Import-Package: com.ibm.icu.text,
  org.eclipse.equinox.http.jetty;resolution:=optional
 Bundle-RequiredExecutionEnvironment: JavaSE-1.8
diff --git a/org.eclipse.help.base/src/org/eclipse/help/internal/search/Analyzer_en.java b/org.eclipse.help.base/src/org/eclipse/help/internal/search/Analyzer_en.java
index b6ccf6f16..b94e0c584 100644
--- a/org.eclipse.help.base/src/org/eclipse/help/internal/search/Analyzer_en.java
+++ b/org.eclipse.help.base/src/org/eclipse/help/internal/search/Analyzer_en.java
@@ -16,11 +16,11 @@ import java.util.HashSet;
 import java.util.Set;
 
 import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.CharArraySet;
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.Tokenizer;
 import org.apache.lucene.analysis.core.StopFilter;
 import org.apache.lucene.analysis.en.PorterStemFilter;
-import org.apache.lucene.analysis.util.CharArraySet;
 
 /**
  * Lucene Analyzer for English.
diff --git a/org.eclipse.help.base/src/org/eclipse/help/internal/search/PluginIndex.java b/org.eclipse.help.base/src/org/eclipse/help/internal/search/PluginIndex.java
index 3bc705188..6aac98619 100644
--- a/org.eclipse.help.base/src/org/eclipse/help/internal/search/PluginIndex.java
+++ b/org.eclipse.help.base/src/org/eclipse/help/internal/search/PluginIndex.java
@@ -152,7 +152,7 @@ public class PluginIndex {
 					.getProperty(SearchIndex.DEPENDENCIES_KEY_ANALYZER);
 			if (!targetIndex.isLuceneCompatible(lucene) || !targetIndex.isAnalyzerCompatible(analyzer)) {
 				String message = "Unable to consume Lucene index from bundle '" + bundle.toString() //$NON-NLS-1$
-						+ "'. The index should be rebuilt with Lucene 6.1."; //$NON-NLS-1$
+						+ "'. The index should be rebuilt with Lucene 7.0.0"; //$NON-NLS-1$
 				Status warningStatus = new Status(IStatus.WARNING, HelpBasePlugin.PLUGIN_ID, IStatus.OK, message, null);
 				HelpBasePlugin.logStatus(warningStatus);
 				return false;
diff --git a/org.eclipse.help.base/src/org/eclipse/help/internal/search/SearchIndex.java b/org.eclipse.help.base/src/org/eclipse/help/internal/search/SearchIndex.java
index 74d17d06c..6085bb53d 100644
--- a/org.eclipse.help.base/src/org/eclipse/help/internal/search/SearchIndex.java
+++ b/org.eclipse.help.base/src/org/eclipse/help/internal/search/SearchIndex.java
@@ -46,11 +46,10 @@ import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.index.IndexWriterConfig;
 import org.apache.lucene.index.IndexWriterConfig.OpenMode;
-import org.apache.lucene.index.LeafReader;
+import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.index.LogByteSizeMergePolicy;
 import org.apache.lucene.index.LogMergePolicy;
 import org.apache.lucene.index.PostingsEnum;
-import org.apache.lucene.index.SlowCompositeReaderWrapper;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.search.BooleanQuery;
 import org.apache.lucene.search.IndexSearcher;
@@ -562,15 +561,18 @@ public class SearchIndex implements IHelpSearchIndex {
 
 	public IStatus removeDuplicates(String name, String[] index_paths) {
 
-		try (LeafReader ar = SlowCompositeReaderWrapper.wrap(ir)) {
+		try (DirectoryReader ar = DirectoryReader.open(luceneDirectory)) {
 			PostingsEnum hrefDocs = null;
 			PostingsEnum indexDocs = null;
 			Term hrefTerm = new Term(FIELD_NAME, name);
 			for (int i = 0; i < index_paths.length; i++) {
 				Term indexTerm = new Term(FIELD_INDEX_ID, index_paths[i]);
-				hrefDocs = ar.postings(hrefTerm);
-				indexDocs = ar.postings(indexTerm);
-				removeDocuments(hrefDocs, indexDocs);
+				List<LeafReaderContext> leaves = ar.leaves();
+				for (LeafReaderContext c : leaves) {
+					indexDocs = c.reader().postings(indexTerm);
+					hrefDocs = c.reader().postings(hrefTerm);
+					removeDocuments(hrefDocs, indexDocs);
+				}
 			}
 		} catch (IOException ioe) {
 			return new Status(IStatus.ERROR, HelpBasePlugin.PLUGIN_ID, IStatus.ERROR,
@@ -750,9 +752,9 @@ public class SearchIndex implements IHelpSearchIndex {
 		}
 		Version luceneVersion = new Version(luceneVersionString);
 		Version indexVersion = new Version(indexVersionString);
-		Version v610 = new Version(6, 1, 0);
-		if (indexVersion.compareTo(v610) < 0) {
-			// index is older than Lucene 6.1.0
+		Version v700 = new Version(7, 0, 0);
+		if (indexVersion.compareTo(v700) < 0) {
+			// index is older than Lucene 7.0.0
 			return false;
 		}
 		if ( luceneVersion.compareTo(indexVersion) >= 0 ) {
diff --git a/org.eclipse.help.webapp/web-archive/org.eclipse.help.infocenter-feature/feature.xml b/org.eclipse.help.webapp/web-archive/org.eclipse.help.infocenter-feature/feature.xml
index 3e5535fc8..787e9dd45 100644
--- a/org.eclipse.help.webapp/web-archive/org.eclipse.help.infocenter-feature/feature.xml
+++ b/org.eclipse.help.webapp/web-archive/org.eclipse.help.infocenter-feature/feature.xml
@@ -234,11 +234,4 @@ IBM Corporation - initial API and implementation
          version="0.0.0"
          unpack="false"/>
 
-   <plugin
-         id="org.apache.lucene.misc"
-         download-size="0"
-         install-size="0"
-         version="0.0.0"
-         unpack="false"/>
-
 </feature>
diff --git a/org.eclipse.ua.tests/data/help/searchindex/index700/_1.fdt b/org.eclipse.ua.tests/data/help/searchindex/index700/_1.fdt
new file mode 100644
index 000000000..f68115115
--- /dev/null
+++ b/org.eclipse.ua.tests/data/help/searchindex/index700/_1.fdt
diff --git a/org.eclipse.ua.tests/data/help/searchindex/index700/_1.fdx b/org.eclipse.ua.tests/data/help/searchindex/index700/_1.fdx
new file mode 100644
index 000000000..b75bd23f7
--- /dev/null
+++ b/org.eclipse.ua.tests/data/help/searchindex/index700/_1.fdx
diff --git a/org.eclipse.ua.tests/data/help/searchindex/index700/_1.fnm b/org.eclipse.ua.tests/data/help/searchindex/index700/_1.fnm
new file mode 100644
index 000000000..703d0571d
--- /dev/null
+++ b/org.eclipse.ua.tests/data/help/searchindex/index700/_1.fnm
diff --git a/org.eclipse.ua.tests/data/help/searchindex/index700/_1.nvd b/org.eclipse.ua.tests/data/help/searchindex/index700/_1.nvd
new file mode 100644
index 000000000..d9d831821
--- /dev/null
+++ b/org.eclipse.ua.tests/data/help/searchindex/index700/_1.nvd
diff --git a/org.eclipse.ua.tests/data/help/searchindex/index700/_1.nvm b/org.eclipse.ua.tests/data/help/searchindex/index700/_1.nvm
new file mode 100644
index 000000000..e4b4ce5a0
--- /dev/null
+++ b/org.eclipse.ua.tests/data/help/searchindex/index700/_1.nvm
diff --git a/org.eclipse.ua.tests/data/help/searchindex/index700/_1.si b/org.eclipse.ua.tests/data/help/searchindex/index700/_1.si
new file mode 100644
index 000000000..5dca9f540
--- /dev/null
+++ b/org.eclipse.ua.tests/data/help/searchindex/index700/_1.si
diff --git a/org.eclipse.ua.tests/data/help/searchindex/index700/_1_Lucene50_0.doc b/org.eclipse.ua.tests/data/help/searchindex/index700/_1_Lucene50_0.doc
new file mode 100644
index 000000000..f7d15c25d
--- /dev/null
+++ b/org.eclipse.ua.tests/data/help/searchindex/index700/_1_Lucene50_0.doc
diff --git a/org.eclipse.ua.tests/data/help/searchindex/index700/_1_Lucene50_0.pos b/org.eclipse.ua.tests/data/help/searchindex/index700/_1_Lucene50_0.pos
new file mode 100644
index 000000000..ffe96a5eb
--- /dev/null
+++ b/org.eclipse.ua.tests/data/help/searchindex/index700/_1_Lucene50_0.pos
diff --git a/org.eclipse.ua.tests/data/help/searchindex/index700/_1_Lucene50_0.tim b/org.eclipse.ua.tests/data/help/searchindex/index700/_1_Lucene50_0.tim
new file mode 100644
index 000000000..86b6f8d02
--- /dev/null
+++ b/org.eclipse.ua.tests/data/help/searchindex/index700/_1_Lucene50_0.tim
diff --git a/org.eclipse.ua.tests/data/help/searchindex/index700/_1_Lucene50_0.tip b/org.eclipse.ua.tests/data/help/searchindex/index700/_1_Lucene50_0.tip
new file mode 100644
index 000000000..c34a4fe89
--- /dev/null
+++ b/org.eclipse.ua.tests/data/help/searchindex/index700/_1_Lucene50_0.tip
diff --git a/org.eclipse.ua.tests/data/help/searchindex/index700/indexed_contributions b/org.eclipse.ua.tests/data/help/searchindex/index700/indexed_contributions
new file mode 100644
index 000000000..b421b5b07
--- /dev/null
+++ b/org.eclipse.ua.tests/data/help/searchindex/index700/indexed_contributions
@@ -0,0 +1,20 @@
+#This is a generated file; do not edit.
+#Mon Nov 07 13:22:27 CET 2016
+org.eclipse.ua.tests=org.eclipse.ua.tests\n3.3.800.qualifier
+org.eclipse.help.base=org.eclipse.help.base\n4.2.0.qualifier
+org.eclipse.jdt=org.eclipse.jdt\n3.13.0.N20161102-2000
+org.eclipse.jdt.doc.user=org.eclipse.jdt.doc.user\n3.13.0.N20161102-2000
+org.eclipse.egit.doc=org.eclipse.egit.doc\n4.4.1.201607150455-r
+org.eclipse.jdt.doc.isv=org.eclipse.jdt.doc.isv\n3.13.0.N20161102-2000
+org.eclipse.ui.cheatsheets=org.eclipse.ui.cheatsheets\n3.5.0.qualifier
+org.eclipse.cvs=org.eclipse.cvs\n1.4.300.N20161102-2000
+org.eclipse.ui.intro=org.eclipse.ui.intro\n3.5.100.qualifier
+org.eclipse.help=org.eclipse.help\n3.8.0.qualifier
+org.eclipse.platform.doc.isv=org.eclipse.platform.doc.isv\n4.7.0.N20161102-2000
+org.eclipse.wst.jsdt.doc=org.eclipse.wst.jsdt.doc\n1.4.101.v201507140011
+org.eclipse.pde.doc.user=org.eclipse.pde.doc.user\n3.13.0.N20161102-2000
+org.eclipse.ui.intro.solstice.examples=org.eclipse.ui.intro.solstice.examples\n1.0.0.qualifier
+org.eclipse.platform=org.eclipse.platform\n4.7.0.N20161102-2000
+org.eclipse.pde=org.eclipse.pde\n3.13.0.N20161102-2000
+org.eclipse.platform.doc.user=org.eclipse.platform.doc.user\n4.7.0.N20161102-2000
+org.eclipse.ui.intro.universal=org.eclipse.ui.intro.universal\n3.3.100.qualifier
diff --git a/org.eclipse.ua.tests/data/help/searchindex/index700/indexed_dependencies b/org.eclipse.ua.tests/data/help/searchindex/index700/indexed_dependencies
new file mode 100644
index 000000000..12594affd
--- /dev/null
+++ b/org.eclipse.ua.tests/data/help/searchindex/index700/indexed_dependencies
@@ -0,0 +1,4 @@
+#This is a generated file; do not edit.
+#Mon Nov 07 13:22:27 CET 2016
+lucene=7.0.0
+analyzer=org.eclipse.help.base\#4.2.0.qualifier?locale\=en
diff --git a/org.eclipse.ua.tests/data/help/searchindex/index700/indexed_docs b/org.eclipse.ua.tests/data/help/searchindex/index700/indexed_docs
new file mode 100644
index 000000000..deac8bd82
--- /dev/null
+++ b/org.eclipse.ua.tests/data/help/searchindex/index700/indexed_docs
@@ -0,0 +1,3 @@
+#This is a generated file; do not edit.
+#Mon Nov 07 13:22:27 CET 2016
+test=true
diff --git a/org.eclipse.ua.tests/data/help/searchindex/index700/segments_2 b/org.eclipse.ua.tests/data/help/searchindex/index700/segments_2
new file mode 100644
index 000000000..4900f1996
--- /dev/null
+++ b/org.eclipse.ua.tests/data/help/searchindex/index700/segments_2
diff --git a/org.eclipse.ua.tests/data/help/searchindex/index700/write.lock b/org.eclipse.ua.tests/data/help/searchindex/index700/write.lock
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/org.eclipse.ua.tests/data/help/searchindex/index700/write.lock
diff --git a/org.eclipse.ua.tests/help/org/eclipse/ua/tests/help/search/PrebuiltIndexCompatibility.java b/org.eclipse.ua.tests/help/org/eclipse/ua/tests/help/search/PrebuiltIndexCompatibility.java
index fccaecec9..5d393bab0 100644
--- a/org.eclipse.ua.tests/help/org/eclipse/ua/tests/help/search/PrebuiltIndexCompatibility.java
+++ b/org.eclipse.ua.tests/help/org/eclipse/ua/tests/help/search/PrebuiltIndexCompatibility.java
@@ -83,12 +83,20 @@ public class PrebuiltIndexCompatibility {
 	/**
 	 * Test index built with Lucene 6.1.0
 	 */
-	@Test
-	public void test6_1_0_IndexReadable() throws Exception {
+	@Test(expected = IllegalArgumentException.class)
+	public void test6_1_0_IndexUnReadable() throws Exception {
 		checkReadable("data/help/searchindex/index610");
 	}
 
 	/**
+	 * Test index built with Lucene 7.0.0
+	 */
+	@Test
+	public void test7_0_0_IndexReadable() throws Exception {
+		checkReadable("data/help/searchindex/index700");
+	}
+
+	/**
 	 ** Test compatibility of Lucene 1.9.1 index with current Lucene
 	 */
 	@Test
@@ -119,7 +127,15 @@ public class PrebuiltIndexCompatibility {
 	 */
 	@Test
 	public void test6_1_0Compatible() {
-		checkCompatible("data/help/searchindex/index610", true);
+		checkCompatible("data/help/searchindex/index610", false);
+	}
+
+	/**
+	 ** Test compatibility of Lucene 7.0.0 index with current Lucene
+	 */
+	@Test
+	public void test7_0_0Compatible() {
+		checkCompatible("data/help/searchindex/index700", true);
 	}
 
 	@Test
@@ -147,7 +163,12 @@ public class PrebuiltIndexCompatibility {
 
 	@Test
 	public void test6_1_0LuceneCompatible() {
-		checkLuceneCompatible("6.1.0", true);
+		checkLuceneCompatible("6.1.0", false);
+	}
+
+	@Test
+	public void test7_0_0LuceneCompatible() {
+		checkLuceneCompatible("7.0.0", true);
 	}
 
 	@Test