Bug 558809: Handle cases where Oomph corrupts \0 char in preference

Some CDT preferences use \0 as a separator in preferences. Somewhere
in the Oomph preference synchronizer stack there is, or was, a place
that failed to escape/unescape preferences with encoded \0 properly.

CDT would then fail to parse the preference and an exception would
be raised, causing code completions and the editor to be broken.

This patch hardens the CDT code to:
(1) Allow an escaped \0 to be used as a separator on
    read (Oomph uses ${0x0})
(2) Handle NumberFormatExceptions gracefully. In this case that means
    showing user a pop-up that their completion preferences
    are empty and offering to reset them, or edit them in preference
    page. This UI logic already existed, so all the new code
    has to do on failed parse is return a list of all disabled
    completions.

Change-Id: Ibf3b05c0855bb96c195ca43139a50c27a2a90c7e

2 files changed, 106 insertions, 21 deletions

diff --git a/core/org.eclipse.cdt.ui/src/org/eclipse/cdt/internal/ui/text/contentassist/CompletionProposalComputerPreferenceParser.java b/core/org.eclipse.cdt.ui/src/org/eclipse/cdt/internal/ui/text/contentassist/CompletionProposalComputerPreferenceParser.java
new file mode 100644
index 00000000000..428baf98a6c
--- /dev/null
+++ b/core/org.eclipse.cdt.ui/src/org/eclipse/cdt/internal/ui/text/contentassist/CompletionProposalComputerPreferenceParser.java
@@ -0,0 +1,78 @@
+/*******************************************************************************
+ * Copyright (c) 2020 Kichwa Coders Canada Inc. and others.
+ *
+ * This program and the accompanying materials
+ * are made available under the terms of the Eclipse Public License 2.0
+ * which accompanies this distribution, and is available at
+ * https://www.eclipse.org/legal/epl-2.0/
+ *
+ * SPDX-License-Identifier: EPL-2.0
+ */
+package org.eclipse.cdt.internal.ui.text.contentassist;
+
+import java.text.ParseException;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+import org.eclipse.cdt.ui.PreferenceConstants;
+
+/**
+ * Parses the Completion Proposal Computer Preferences.
+ * <p>
+ * See org.eclipse.cdt.internal.ui.preferences.CodeAssistAdvancedConfigurationBlock.PreferenceModel
+ * for the write side of the preferences.
+ */
+public class CompletionProposalComputerPreferenceParser {
+	/**
+	 * Parses the {@link PreferenceConstants#CODEASSIST_EXCLUDED_CATEGORIES} value and
+	 * converts to a set of categories that are excluded.
+	 * @param preferenceValue as stored in {@link PreferenceConstants#CODEASSIST_EXCLUDED_CATEGORIES}
+	 * @return set of excluded categories
+	 * @throws ParseException if the value cannot be parsed
+	 */
+	public static Set<String> parseExcludedCategories(String preferenceValue) throws ParseException {
+		Set<String> disabled = new HashSet<>();
+		String[] disabledArray = splitOnNulls(preferenceValue);
+		disabled.addAll(Arrays.asList(disabledArray));
+		return disabled;
+	}
+
+	/**
+	 * Parses the {@link PreferenceConstants#CODEASSIST_CATEGORY_ORDER} value and
+	 * converts to a map of category ids to sort rank number
+	 * @param preferenceValue as stored in {@link PreferenceConstants#CODEASSIST_CATEGORY_ORDER}
+	 * @return map of category id to rank order
+	 * @throws ParseException if the value cannot be parsed
+	 */
+	public static Map<String, Integer> parseCategoryOrder(String preferenceValue) throws ParseException {
+		Map<String, Integer> ordered = new HashMap<>();
+		String[] orderedArray = splitOnNulls(preferenceValue);
+		for (String entry : orderedArray) {
+			String[] idRank = entry.split(":"); //$NON-NLS-1$
+			if (idRank.length != 2) {
+				throw new ParseException(entry, 0);
+			}
+			String id = idRank[0];
+			int rank;
+			try {
+				rank = Integer.parseInt(idRank[1]);
+			} catch (NumberFormatException e) {
+				throw new ParseException(entry, 0);
+			}
+			ordered.put(id, Integer.valueOf(rank));
+		}
+		return ordered;
+	}
+
+	/**
+	 * See Bug 558809. Oomph seems to have failed at times to encode/decode nul character '\0'
+	 * from the format it is stored in Oomph setup files. We can have ${0x0} instead of \0, infact
+	 * there can be multiple $, so $$$$${0x0} is a valid split too.
+	 */
+	private static String[] splitOnNulls(String preference) {
+		return preference.split("\\000|(\\$+\\{0x0\\})"); //$NON-NLS-1$
+	}
+}
diff --git a/core/org.eclipse.cdt.ui/src/org/eclipse/cdt/internal/ui/text/contentassist/CompletionProposalComputerRegistry.java b/core/org.eclipse.cdt.ui/src/org/eclipse/cdt/internal/ui/text/contentassist/CompletionProposalComputerRegistry.java
index 12c66d662e1..33843a56842 100644
--- a/core/org.eclipse.cdt.ui/src/org/eclipse/cdt/internal/ui/text/contentassist/CompletionProposalComputerRegistry.java
+++ b/core/org.eclipse.cdt.ui/src/org/eclipse/cdt/internal/ui/text/contentassist/CompletionProposalComputerRegistry.java
@@ -14,6 +14,7 @@
  *******************************************************************************/
 package org.eclipse.cdt.internal.ui.text.contentassist;
 
+import java.text.ParseException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
@@ -24,7 +25,6 @@ import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
-import java.util.StringTokenizer;
 
 import org.eclipse.cdt.internal.ui.util.Messages;
 import org.eclipse.cdt.ui.CUIPlugin;
@@ -272,19 +272,19 @@ public final class CompletionProposalComputerRegistry {
 
 	private List<CompletionProposalCategory> getCategories(List<IConfigurationElement> elements) {
 		IPreferenceStore store = CUIPlugin.getDefault().getPreferenceStore();
-		String preference = store.getString(PreferenceConstants.CODEASSIST_EXCLUDED_CATEGORIES);
-		Set<String> disabled = new HashSet<>();
-		StringTokenizer tok = new StringTokenizer(preference, "\0"); //$NON-NLS-1$
-		while (tok.hasMoreTokens())
-			disabled.add(tok.nextToken());
-		Map<String, Integer> ordered = new HashMap<>();
-		preference = store.getString(PreferenceConstants.CODEASSIST_CATEGORY_ORDER);
-		tok = new StringTokenizer(preference, "\0"); //$NON-NLS-1$
-		while (tok.hasMoreTokens()) {
-			StringTokenizer inner = new StringTokenizer(tok.nextToken(), ":"); //$NON-NLS-1$
-			String id = inner.nextToken();
-			int rank = Integer.parseInt(inner.nextToken());
-			ordered.put(id, Integer.valueOf(rank));
+		Set<String> disabled = Collections.emptySet();
+		Map<String, Integer> ordered = Collections.emptyMap();
+		boolean parseFailed = false;
+		try {
+			disabled = CompletionProposalComputerPreferenceParser
+					.parseExcludedCategories(store.getString(PreferenceConstants.CODEASSIST_EXCLUDED_CATEGORIES));
+			ordered = CompletionProposalComputerPreferenceParser
+					.parseCategoryOrder(store.getString(PreferenceConstants.CODEASSIST_CATEGORY_ORDER));
+		} catch (ParseException e) {
+			// Failed to parse user setting, clear all settings
+			// and display error message to user allowing them to
+			// reset on first use
+			parseFailed = true;
 		}
 
 		List<CompletionProposalCategory> categories = new ArrayList<>();
@@ -296,13 +296,20 @@ public final class CompletionProposalComputerRegistry {
 
 					CompletionProposalCategory category = new CompletionProposalCategory(element, this);
 					categories.add(category);
-					category.setIncluded(!disabled.contains(category.getId()));
-					Integer rank = ordered.get(category.getId());
-					if (rank != null) {
-						int r = rank.intValue();
-						boolean separate = r < 0xffff;
-						category.setSeparateCommand(separate);
-						category.setSortOrder(r);
+					if (parseFailed) {
+						// When parse has failed we do the same thing as if the user had disabled
+						// off ever proposal category. This causes a pop-up on first completion
+						// attempt with the option of resetting defaults
+						category.setIncluded(false);
+					} else {
+						category.setIncluded(!disabled.contains(category.getId()));
+						Integer rank = ordered.get(category.getId());
+						if (rank != null) {
+							int r = rank.intValue();
+							boolean separate = r < 0xffff;
+							category.setSeparateCommand(separate);
+							category.setSortOrder(r);
+						}
 					}
 				}
 			} catch (CoreException x) {