Bug 534073: [Tips] Create JSon based Tip Provider for Platform

* JSon Provider created for Platform
* Tips are uploaded to E4 area
* Fixed some read state and provider caching

Change-Id: I3566e0f440e44777a3d89cceed656aa0c8c3634e
Signed-off-by: Wim Jongman <wim.jongman@remainsoftware.com>

7 files changed, 271 insertions, 39 deletions

diff --git a/org.eclipse.tips.ide/META-INF/MANIFEST.MF b/org.eclipse.tips.ide/META-INF/MANIFEST.MF
index 35e3add48..6e09db3a6 100644
--- a/org.eclipse.tips.ide/META-INF/MANIFEST.MF
+++ b/org.eclipse.tips.ide/META-INF/MANIFEST.MF
@@ -9,6 +9,7 @@ Require-Bundle: org.eclipse.ui;bundle-version="3.109.0",
  org.eclipse.core.expressions;bundle-version="3.6.0",
  org.eclipse.e4.ui.workbench;bundle-version="1.5.1",
  org.eclipse.tips.core;bundle-version="0.1.0",
- org.eclipse.tips.ui;bundle-version="0.1.0"
+ org.eclipse.tips.ui;bundle-version="0.1.0",
+ org.eclipse.tips.json
 Export-Package: org.eclipse.tips.ide.internal;x-internal:=true
 Automatic-Module-Name: org.eclipse.tips.ide
diff --git a/org.eclipse.tips.ide/src/org/eclipse/tips/ide/internal/IDETipManager.java b/org.eclipse.tips.ide/src/org/eclipse/tips/ide/internal/IDETipManager.java
index 728b10611..5cd8f4d55 100644
--- a/org.eclipse.tips.ide/src/org/eclipse/tips/ide/internal/IDETipManager.java
+++ b/org.eclipse.tips.ide/src/org/eclipse/tips/ide/internal/IDETipManager.java
@@ -11,6 +11,8 @@
 package org.eclipse.tips.ide.internal;
 
 import java.io.ByteArrayInputStream;
+import java.io.File;
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
@@ -99,7 +101,7 @@ public class IDETipManager extends DefaultTipManager {
 			evaluationService.addSourceProvider(fSourceProvider);
 			fSourceProviderAdded = true;
 		}
-		fReadTips = Preferences.getReadState();
+		fReadTips = TipsPreferences.getReadState();
 		return super.open(startUp);
 	}
 
@@ -115,7 +117,7 @@ public class IDETipManager extends DefaultTipManager {
 			protected IStatus run(IProgressMonitor monitor) {
 				SubMonitor subMonitor = SubMonitor.convert(monitor, IProgressMonitor.UNKNOWN);
 				subMonitor.setTaskName("Saving read tips..");
-				IStatus status = Preferences.saveReadState(pReadTips);
+				IStatus status = TipsPreferences.saveReadState(pReadTips);
 				subMonitor.done();
 				return status;
 			}
@@ -143,23 +145,23 @@ public class IDETipManager extends DefaultTipManager {
 
 	@Override
 	public boolean isRunAtStartup() {
-		return Preferences.isRunAtStartup();
+		return TipsPreferences.isRunAtStartup();
 	}
 
 	@Override
 	public TipManager setRunAtStartup(boolean runAtStartup) {
-		Preferences.setRunAtStartup(runAtStartup);
+		TipsPreferences.setRunAtStartup(runAtStartup);
 		return this;
 	}
 
 	@Override
 	public boolean mustServeReadTips() {
-		return Preferences.isServeReadTips();
+		return TipsPreferences.isServeReadTips();
 	}
 
 	@Override
 	public TipManager setServeReadTips(boolean serveRead) {
-		Preferences.setServeReadTips(serveRead);
+		TipsPreferences.setServeReadTips(serveRead);
 		return this;
 	}
 
@@ -267,4 +269,29 @@ public class IDETipManager extends DefaultTipManager {
 		IEvaluationContext currentState = evalService.getCurrentState();
 		return currentState;
 	}
+
+	/**
+	 * Returns the state location of the IDE tips. First the property
+	 * "org.eclipse.tips.statelocation" is read. If it does not exist then the state
+	 * location will be <b>${user.home}/.eclipse/org.eclipse.tips.state</b>
+	 * 
+	 * @return the state location file
+	 * @throws Exception if something went wrong
+	 */
+	public static File getStateLocation() throws Exception {
+		String stateLocation = System.getProperty("org.eclipse.tips.statelocation");
+		if (stateLocation == null) {
+			stateLocation = System.getProperty("user.home") + File.separator + ".eclipse" + File.separator
+					+ "org.eclipse.tips.state";
+		}
+		File locationDir = new File(stateLocation);
+		if (!locationDir.exists()) {
+			locationDir.mkdirs();
+		}
+
+		if (!locationDir.canRead() || !locationDir.canWrite()) {
+			throw new IOException("Could not read or write the state location: " + locationDir.getAbsolutePath());
+		}
+		return locationDir;
+	}
 }
\ No newline at end of file
diff --git a/org.eclipse.tips.ide/src/org/eclipse/tips/ide/internal/Startup.java b/org.eclipse.tips.ide/src/org/eclipse/tips/ide/internal/Startup.java
index dc560e0ed..6dd253b62 100644
--- a/org.eclipse.tips.ide/src/org/eclipse/tips/ide/internal/Startup.java
+++ b/org.eclipse.tips.ide/src/org/eclipse/tips/ide/internal/Startup.java
@@ -17,6 +17,7 @@ import org.eclipse.core.runtime.IStatus;
 import org.eclipse.core.runtime.Platform;
 import org.eclipse.core.runtime.Status;
 import org.eclipse.tips.core.TipProvider;
+import org.eclipse.tips.json.internal.ProviderLoader;
 import org.eclipse.ui.IStartup;
 import org.eclipse.ui.PlatformUI;
 import org.eclipse.ui.progress.UIJob;
@@ -27,6 +28,7 @@ import org.osgi.framework.FrameworkUtil;
  * Early startup to run the TipManager in the IDE.
  *
  */
+@SuppressWarnings("restriction")
 public class Startup implements IStartup {
 
 	@Override
@@ -39,6 +41,25 @@ public class Startup implements IStartup {
 	 * Reloads the tip providers.
 	 */
 	public static void loadProviders() {
+		loadExternalProviders();
+		loadInternalProviders();
+	}
+
+	private static void loadExternalProviders() {
+		String baseURL = System.getProperty("org.eclipse.tips.ide.provider.url");
+		if (baseURL == null) {
+			baseURL = "http://www.eclipse.org/downloads/download.php?r=1&file=/e4/tips/";
+		}
+		try {
+			ProviderLoader.loadProviderData(IDETipManager.getInstance(), baseURL, IDETipManager.getStateLocation());
+		} catch (Exception e) {
+			IDETipManager.getInstance()
+					.log(new Status(IStatus.ERROR, FrameworkUtil.getBundle(Startup.class).getSymbolicName(),
+							"Failure getting the Tips state location.", e));
+		}
+	}
+
+	private static void loadInternalProviders() {
 		IConfigurationElement[] elements = Platform.getExtensionRegistry()
 				.getConfigurationElementsFor("org.eclipse.tips.core.tips");
 		for (IConfigurationElement element : elements) {
diff --git a/org.eclipse.tips.ide/src/org/eclipse/tips/ide/internal/Preferences.java b/org.eclipse.tips.ide/src/org/eclipse/tips/ide/internal/TipsPreferences.java
index 02fb09a00..13bccf699 100644
--- a/org.eclipse.tips.ide/src/org/eclipse/tips/ide/internal/Preferences.java
+++ b/org.eclipse.tips.ide/src/org/eclipse/tips/ide/internal/TipsPreferences.java
@@ -10,6 +10,8 @@
  *******************************************************************************/
 package org.eclipse.tips.ide.internal;
 
+import java.io.File;
+import java.io.FilenameFilter;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
@@ -21,6 +23,7 @@ import org.eclipse.core.runtime.Status;
 import org.eclipse.core.runtime.preferences.AbstractPreferenceInitializer;
 import org.eclipse.core.runtime.preferences.ConfigurationScope;
 import org.eclipse.core.runtime.preferences.IEclipsePreferences;
+import org.eclipse.jface.preference.PreferenceStore;
 import org.osgi.framework.Bundle;
 import org.osgi.framework.FrameworkUtil;
 import org.osgi.service.prefs.BackingStoreException;
@@ -29,7 +32,7 @@ import org.osgi.service.prefs.BackingStoreException;
  * Internal class to store preferences.
  *
  */
-public class Preferences extends AbstractPreferenceInitializer {
+public class TipsPreferences extends AbstractPreferenceInitializer {
 
 	/**
 	 * Preference store key to indicate showing tips at startup.
@@ -41,7 +44,7 @@ public class Preferences extends AbstractPreferenceInitializer {
 	 */
 	public static final String PREF_SERVE_READ_TIPS = "serve_read_tips";
 
-	public Preferences() {
+	public TipsPreferences() {
 	}
 
 	@Override
@@ -63,23 +66,47 @@ public class Preferences extends AbstractPreferenceInitializer {
 	 */
 	public static Map<String, List<Integer>> getReadState() {
 		HashMap<String, List<Integer>> result = new HashMap<>();
-		IEclipsePreferences node = ConfigurationScope.INSTANCE.getNode("org.eclipse.tips.ide.read");
+
 		try {
-			for (String key : node.childrenNames()) {
+			File stateLocation = getStateLocation();
+			for (String key : stateLocation.list(getStateFileNameFilter(stateLocation))) {
+				PreferenceStore store = new PreferenceStore(new File(stateLocation, key).getAbsolutePath());
+				store.load();
 				ArrayList<Integer> tips = new ArrayList<>();
-				org.osgi.service.prefs.Preferences tipsNode = node.node(key);
-				for (String tipKey : tipsNode.keys()) {
-					tips.add(Integer.valueOf(tipsNode.getInt(tipKey, 0)));
+				for (String tipKey : store.preferenceNames()) {
+					if (!"provider".equals(tipKey)) {
+						tips.add(Integer.valueOf(store.getInt(tipKey)));
+					}
 				}
-				result.put(key, tips);
+				result.put(store.getString("provider"), tips);
 			}
-		} catch (BackingStoreException e) {
+		} catch (Exception e) {
 			Status status = new Status(IStatus.ERROR, "org.eclipse.tips.ide", e.getMessage(), e);
 			log(status);
 		}
 		return result;
 	}
 
+	private static FilenameFilter getStateFileNameFilter(File stateLocation) {
+		return new FilenameFilter() {
+			@Override
+			public boolean accept(File pDir, String pName) {
+				if (pDir.equals(stateLocation) && pName.endsWith(".state")) {
+					return true;
+				}
+				return false;
+			}
+		};
+	}
+
+	private static File getStateLocation() throws Exception {
+		File file = new File(IDETipManager.getStateLocation(), "org.eclipse.tips.ide.state");
+		if (!file.exists()) {
+			file.mkdirs();
+		}
+		return file;
+	}
+
 	/**
 	 * Saves the list with read tips to disk.
 	 * 
@@ -88,18 +115,16 @@ public class Preferences extends AbstractPreferenceInitializer {
 	 */
 	public static IStatus saveReadState(Map<String, List<Integer>> pReadTips) {
 		try {
-			IEclipsePreferences node = ConfigurationScope.INSTANCE.getNode("org.eclipse.tips.ide.read");
+			File stateLocation = getStateLocation();
 			for (String child : pReadTips.keySet()) {
-				if (node.nodeExists(child)) {
-					node.node(child).removeNode();
-				}
+				PreferenceStore store = new PreferenceStore(
+						new File(stateLocation, child.trim() + ".state").getAbsolutePath());
+				pReadTips.get(child).forEach(value -> store.setValue(value.toString(), value.intValue()));
+				store.setValue("provider", child);
+				store.save();
 			}
-			node.clear();
-			pReadTips.forEach(
-					(key, tips) -> tips.forEach(value -> node.node(key).putInt(value.toString(), value.intValue())));
-			node.flush();
 			return Status.OK_STATUS;
-		} catch (BackingStoreException e) {
+		} catch (Exception e) {
 			e.printStackTrace();
 			return new Status(IStatus.ERROR, "org.eclipse.tips.ide", e.getMessage(), e);
 		}
@@ -112,7 +137,7 @@ public class Preferences extends AbstractPreferenceInitializer {
 
 	private static void log(IStatus status) {
 		if (status.matches(IStatus.ERROR | IStatus.WARNING)) {
-			Bundle bundle = FrameworkUtil.getBundle(Preferences.class);
+			Bundle bundle = FrameworkUtil.getBundle(TipsPreferences.class);
 			Platform.getLog(bundle).log(status);
 		}
 		if (System.getProperty("org.eclipse.tips.consolelog") != null) {
diff --git a/org.eclipse.tips.json/src/org/eclipse/tips/json/JsonTipProvider.java b/org.eclipse.tips.json/src/org/eclipse/tips/json/JsonTipProvider.java
index 3a303b37d..c9286268a 100644
--- a/org.eclipse.tips.json/src/org/eclipse/tips/json/JsonTipProvider.java
+++ b/org.eclipse.tips.json/src/org/eclipse/tips/json/JsonTipProvider.java
@@ -10,6 +10,7 @@
  *******************************************************************************/
 package org.eclipse.tips.json;
 
+import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.net.MalformedURLException;
 import java.net.URL;
@@ -49,10 +50,8 @@ public abstract class JsonTipProvider extends TipProvider {
 	 * A method to set the a url containing a JSon file that describes this tip
 	 * provider.
 	 *
-	 * @param jsonUrl
-	 *            the uRL to the Json file describing the provider and tips
-	 * @throws MalformedURLException
-	 *             in case of an incorrect URL
+	 * @param jsonUrl the uRL to the Json file describing the provider and tips
+	 * @throws MalformedURLException in case of an incorrect URL
 	 */
 	public void setJsonUrl(String jsonUrl) throws MalformedURLException {
 		fJsonUrl = new URL(jsonUrl);
@@ -64,14 +63,16 @@ public abstract class JsonTipProvider extends TipProvider {
 		ArrayList<Tip> result = new ArrayList<>();
 		try {
 			subMonitor.beginTask(getDescription() + " Loading Tips", -1);
-			JsonObject value = (JsonObject) new JsonParser().parse(new InputStreamReader(fJsonUrl.openStream()));
-			JsonObject provider = value.getAsJsonObject(JsonConstants.P_PROVIDER);
-			fDescription = Util.getValueOrDefault(provider, JsonConstants.P_DESCRIPTION, "not set");
-			fImage = Util.getValueOrDefault(provider, JsonConstants.P_IMAGE, null);
-			setExpression(Util.getValueOrDefault(provider, JsonConstants.P_EXPRESSION, null));
-			JsonArray tips = provider.getAsJsonArray(JsonConstants.P_TIPS);
-			subMonitor.beginTask(getDescription() + " Creating Tips", -1);
-			tips.forEach(parm -> result.add(createJsonTip(parm)));
+			try (InputStream stream = fJsonUrl.openStream(); InputStreamReader reader = new InputStreamReader(stream)) {
+				JsonObject value = (JsonObject) new JsonParser().parse(reader);
+				JsonObject provider = value.getAsJsonObject(JsonConstants.P_PROVIDER);
+				fDescription = Util.getValueOrDefault(provider, JsonConstants.P_DESCRIPTION, "not set");
+				fImage = Util.getValueOrDefault(provider, JsonConstants.P_IMAGE, null);
+				setExpression(Util.getValueOrDefault(provider, JsonConstants.P_EXPRESSION, null));
+				JsonArray tips = provider.getAsJsonArray(JsonConstants.P_TIPS);
+				subMonitor.beginTask(getDescription() + " Creating Tips", -1);
+				tips.forEach(parm -> result.add(createJsonTip(parm)));
+			}
 		} catch (Exception e) {
 			Status status = new Status(IStatus.ERROR, "org.eclipse.tips.json", e.getMessage(), e);
 			getManager().log(status);
diff --git a/org.eclipse.tips.json/src/org/eclipse/tips/json/internal/ProviderLoader.java b/org.eclipse.tips.json/src/org/eclipse/tips/json/internal/ProviderLoader.java
new file mode 100644
index 000000000..2173001fe
--- /dev/null
+++ b/org.eclipse.tips.json/src/org/eclipse/tips/json/internal/ProviderLoader.java
@@ -0,0 +1,157 @@
+/*******************************************************************************
+ * Copyright (c) 2018 Remain Software
+ * All rights reserved. This program and the accompanying materials
+ * are made available under the terms of the Eclipse Public License v1.0
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/epl-v10.html
+ *
+ * Contributors:
+ *     wim.jongman@remainsoftware.com - initial API and implementation
+ *******************************************************************************/
+package org.eclipse.tips.json.internal;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.FileReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.nio.file.Files;
+import java.nio.file.StandardCopyOption;
+
+import org.eclipse.core.runtime.IStatus;
+import org.eclipse.core.runtime.Platform;
+import org.eclipse.core.runtime.Status;
+import org.eclipse.tips.core.ITipManager;
+import org.eclipse.tips.json.JsonTipProvider;
+import org.osgi.framework.Bundle;
+import org.osgi.framework.FrameworkUtil;
+
+import com.google.gson.JsonArray;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonObject;
+import com.google.gson.JsonParser;
+
+/**
+ * A helper class to load providers from an internet location.
+ *
+ */
+public class ProviderLoader {
+
+	/**
+	 * This method loads provider information from the internet and stores it
+	 * locally. This method should not be called in the user interface thread.
+	 * 
+	 * @param pManager      the tip manager
+	 * @param baseURL       the location of the providers.json file
+	 * @param stateLocation the place to store state
+	 */
+	public static void loadProviderData(ITipManager pManager, String baseURL, File stateLocation) {
+		loadProviders(pManager, baseURL, stateLocation);
+	}
+
+	private static void loadProviders(ITipManager pManager, String pBaseURL, File stateLocation) {
+		try {
+			URL webFile = new URL(pBaseURL + "providers.json");
+			File target = new File(stateLocation, "providers.json");
+			try (InputStream in = webFile.openStream()) {
+				Files.copy(in, target.toPath(), StandardCopyOption.REPLACE_EXISTING);
+			}
+			createProviders(pManager, target, pBaseURL, stateLocation);
+		} catch (Exception e) {
+			String symbolicName = FrameworkUtil.getBundle(ProviderLoader.class).getSymbolicName();
+			pManager.log(new Status(IStatus.ERROR, symbolicName, "Error loading provider file", e));
+		}
+	}
+
+	private static void createProviders(ITipManager pManager, File pTarget, String pBaseURL, File stateLocation)
+			throws Exception {
+		try (FileReader reader = new FileReader(pTarget)) {
+			JsonObject value = (JsonObject) new JsonParser().parse(reader);
+			JsonArray providers = value.getAsJsonArray(JsonConstants.P_PROVIDER);
+			providers.forEach(provider -> loadProvider(pManager, provider, pBaseURL, stateLocation));
+		}
+	}
+
+	private static void loadProvider(ITipManager pManager, JsonElement pProvider, String pBaseURL, File userLocation) {
+		JsonObject provider = pProvider.getAsJsonObject();
+		String version = Util.getValueOrDefault(provider, "version", null);
+		String bundleName = Util.getValueOrDefault(provider, "require-bundle", null);
+		if (version == null || bundleName == null) {
+			logInvalidProvider(pManager, provider);
+			return;
+		}
+
+		Bundle bundle = Platform.getBundle(bundleName);
+		if (bundle == null) {
+			logInvalidProvider(pManager, provider);
+			return;
+		}
+
+		try {
+			File fileLocation = new File(userLocation, bundleName);
+			if (!fileLocation.exists()) {
+				fileLocation.mkdirs();
+			}
+
+			File versionFile = new File(fileLocation, "version.txt");
+			if (!versionFile.exists()) {
+				versionFile.createNewFile();
+				try (FileOutputStream fos = new FileOutputStream(versionFile)) {
+					fos.write(version.getBytes());
+				}
+			}
+
+			String existingVersion = getFileContent(versionFile);
+			File providerFile = new File(fileLocation, "provider.json");
+			if (!version.equals(existingVersion) || !providerFile.exists()) {
+				URL webFile = new URL(pBaseURL + "/" + bundleName + "/provider.json");
+				try (InputStream in = webFile.openStream()) {
+					Files.copy(in, providerFile.toPath(), StandardCopyOption.REPLACE_EXISTING);
+				}
+				try (FileOutputStream fos = new FileOutputStream(versionFile)) {
+					fos.write(version.getBytes());
+				}
+			}
+			registerProvider(pManager, bundleName, fileLocation);
+
+		} catch (IOException e) {
+			String symbolicName = FrameworkUtil.getBundle(ProviderLoader.class).getSymbolicName();
+			pManager.log(
+					new Status(IStatus.ERROR, symbolicName, "Error loading provider from: " + pProvider.toString(), e));
+		}
+	}
+
+	private static void registerProvider(ITipManager pManager, String bundleName, File pFileLocation)
+			throws IOException, MalformedURLException {
+		File fileLocation;
+		JsonTipProvider tipProvider = new JsonTipProvider() {
+
+			@Override
+			public String getID() {
+				return bundleName + ".json.provider";
+			}
+		};
+
+		fileLocation = new File(pFileLocation, "provider.json");
+		tipProvider.setJsonUrl(fileLocation.toURI().toURL().toString());
+		pManager.register(tipProvider);
+	}
+
+	public static String getFileContent(File pVersionFile) throws IOException {
+		try (FileInputStream fis = new FileInputStream(pVersionFile)) {
+			try (BufferedReader br = new BufferedReader(new InputStreamReader(fis))) {
+				return br.readLine();
+			}
+		}
+	}
+
+	private static void logInvalidProvider(ITipManager pManager, JsonObject pProvider) {
+		String symbolicName = FrameworkUtil.getBundle(ProviderLoader.class).getSymbolicName();
+		pManager.log(new Status(IStatus.ERROR, symbolicName, "Error loading provider from: " + pProvider.toString()));
+	}
+}
\ No newline at end of file
diff --git a/org.eclipse.tips.ui/src/org/eclipse/tips/ui/internal/DefaultTipManager.java b/org.eclipse.tips.ui/src/org/eclipse/tips/ui/internal/DefaultTipManager.java
index cbb5ff483..a2883e08e 100644
--- a/org.eclipse.tips.ui/src/org/eclipse/tips/ui/internal/DefaultTipManager.java
+++ b/org.eclipse.tips.ui/src/org/eclipse/tips/ui/internal/DefaultTipManager.java
@@ -71,7 +71,7 @@ public abstract class DefaultTipManager extends TipManager {
 				}
 			}
 		}
-		return false;
+		return true;
 	}
 
 }
\ No newline at end of file