Bug 422054: Create profile from existing bundles.infoI20131224-0800

I've hijacked the bug for the purpose of something that is remotely
connected to the problem.

Previous commit extends simpleconfigurator to load bundles from
different locations, but such an extension breaks p2, as the content of
OSGi application no longer corresponds to profile.

In order to fix that, following things are done:
0. A fragment must contain valid P2 metadata next to the .info file.
1. User profile is dropped when there is master configuration change.
Also, when fragments are changed.
2. When a new surrogate profile is created, P2 will add to the master
profile all the units that are found in the fragments repos. Therefore
new user profile will contain all the units that were installed by the
simpleconfigurator. Fragment repos must be runnable.
3. The director application must be configured with extensions if it is
supposed to work.

Change-Id: I6e857ea51dd32ae7fab39d9c39bec8a91eb203b7
Signed-off-by: Krzysztof Daniel <kdaniel@redhat.com>

6 files changed, 250 insertions, 11 deletions

diff --git a/bundles/org.eclipse.equinox.p2.engine/META-INF/MANIFEST.MF b/bundles/org.eclipse.equinox.p2.engine/META-INF/MANIFEST.MF
index 20789fc0e..e44440bad 100644
--- a/bundles/org.eclipse.equinox.p2.engine/META-INF/MANIFEST.MF
+++ b/bundles/org.eclipse.equinox.p2.engine/META-INF/MANIFEST.MF
@@ -8,11 +8,12 @@ Bundle-Vendor: %providerName
 Bundle-Localization: plugin
 Export-Package: org.eclipse.equinox.internal.p2.engine;
   x-friends:="org.eclipse.equinox.p2.garbagecollector,
+   org.eclipse.equinox.p2.repository.tools,
    org.eclipse.equinox.p2.touchpoint.eclipse,
    org.eclipse.equinox.p2.touchpoint.natives,
    org.eclipse.equinox.p2.ui.sdk.scheduler,
    org.eclipse.pde.build,
-   org.eclipse.equinox.p2.repository.tools",
+   org.eclipse.equinox.p2.director.app",
  org.eclipse.equinox.internal.p2.engine.phases;x-friends:="org.eclipse.equinox.p2.director.app,org.eclipse.equinox.p2.repository.tools,org.eclipse.equinox.p2.ui.sdk.scheduler",
  org.eclipse.equinox.p2.engine;version="2.2.0",
  org.eclipse.equinox.p2.engine.query;version="2.0.0",
diff --git a/bundles/org.eclipse.equinox.p2.engine/src/org/eclipse/equinox/internal/p2/engine/EngineActivator.java b/bundles/org.eclipse.equinox.p2.engine/src/org/eclipse/equinox/internal/p2/engine/EngineActivator.java
index 3a96e8590..5fc0e73b5 100644
--- a/bundles/org.eclipse.equinox.p2.engine/src/org/eclipse/equinox/internal/p2/engine/EngineActivator.java
+++ b/bundles/org.eclipse.equinox.p2.engine/src/org/eclipse/equinox/internal/p2/engine/EngineActivator.java
@@ -1,5 +1,5 @@
 /*******************************************************************************
- *  Copyright (c) 2007, 2010 IBM Corporation and others.
+ *  Copyright (c) 2007, 2013 IBM Corporation and others.
  *  All rights reserved. This program and the accompanying materials
  *  are made available under the terms of the Eclipse Public License v1.0
  *  which accompanies this distribution, and is available at
@@ -7,14 +7,33 @@
  * 
  *  Contributors:
  *     IBM Corporation - initial API and implementation
+ *     Red Hat, Inc - fragments support added
  *******************************************************************************/
 package org.eclipse.equinox.internal.p2.engine;
 
+import java.io.*;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.*;
+import org.eclipse.core.runtime.IStatus;
+import org.eclipse.core.runtime.Status;
+import org.eclipse.equinox.internal.p2.core.helpers.LogHelper;
+import org.eclipse.osgi.util.NLS;
 import org.osgi.framework.BundleActivator;
 import org.osgi.framework.BundleContext;
 
 public class EngineActivator implements BundleActivator {
 	private static BundleContext context;
+	/**
+	 * as for now it is exactly the same variable as in simpleconfigurator Activator
+	 * @noreference This field is not intended to be referenced by clients.
+	 */
+	public static String EXTENSIONS = System.getProperty("p2.fragments"); //$NON-NLS-1$
+	public static boolean EXTENDED = EXTENSIONS != null;
+	private static final String LINK_KEY = "link"; //$NON-NLS-1$
+	private static final String LINK_FILE_EXTENSION = ".link"; //$NON-NLS-1$
+	private static final Set<File> reportedExtensions = Collections.synchronizedSet(new HashSet<File>(0));
+
 	public static final String ID = "org.eclipse.equinox.p2.engine"; //$NON-NLS-1$
 
 	/**
@@ -56,6 +75,11 @@ public class EngineActivator implements BundleActivator {
 		return context;
 	}
 
+	/**
+	 * This property indicates repositories that are passed via the fragments mechanism.
+	 */
+	public static final String P2_FRAGMENT_PROPERTY = "p2.fragment"; //$NON-NLS-1$
+
 	public void start(BundleContext aContext) throws Exception {
 		EngineActivator.context = aContext;
 	}
@@ -64,4 +88,83 @@ public class EngineActivator implements BundleActivator {
 		EngineActivator.context = null;
 	}
 
+	public static File[] getExtensionsDirectories() {
+		List<File> files = new ArrayList<File>(0);
+		if (EXTENSIONS != null) {
+			String[] locationToCheck = EXTENSIONS.split(","); //$NON-NLS-1$
+			for (String location : locationToCheck) {
+				try {
+					files.addAll(getInfoFilesFromLocation(location));
+				} catch (FileNotFoundException e) {
+					LogHelper.log(new Status(IStatus.ERROR, ID, NLS.bind(Messages.EngineActivator_0, location), e));
+				} catch (IOException e) {
+					LogHelper.log(new Status(IStatus.ERROR, ID, NLS.bind(Messages.EngineActivator_0, location), e));
+				} catch (URISyntaxException e) {
+					LogHelper.log(new Status(IStatus.ERROR, ID, NLS.bind(Messages.EngineActivator_0, location), e));
+				}
+			}
+		}
+		return files.toArray(new File[files.size()]);
+	}
+
+	// This method must match the implementation in the SimpleConfiguratorUtils with the only difference that
+	// parent folder of the metadata is returned.
+	private static ArrayList<File> getInfoFilesFromLocation(String locationToCheck) throws IOException, FileNotFoundException, URISyntaxException {
+		ArrayList<File> result = new ArrayList<File>(1);
+
+		File extensionsLocation = new File(locationToCheck);
+
+		if (extensionsLocation.exists() && extensionsLocation.isDirectory()) {
+			//extension location contains extensions
+			File[] extensions = extensionsLocation.listFiles();
+			for (File extension : extensions) {
+				if (extension.isFile() && extension.getName().endsWith(LINK_FILE_EXTENSION)) {
+					Properties link = new Properties();
+					link.load(new FileInputStream(extension));
+					String newInfoName = link.getProperty(LINK_KEY);
+					URI newInfoURI = new URI(newInfoName);
+					File newInfoFile = null;
+					if (newInfoURI.isAbsolute()) {
+						newInfoFile = new File(newInfoName);
+					} else {
+						newInfoFile = new File(extension.getParentFile(), newInfoName);
+					}
+					if (newInfoFile.exists()) {
+						extension = newInfoFile.getParentFile();
+					}
+				}
+
+				if (extension.isDirectory()) {
+					if (extension.canWrite()) {
+						synchronized (reportedExtensions) {
+							if (!reportedExtensions.contains(extension)) {
+								reportedExtensions.add(extension);
+								LogHelper.log(new Status(IStatus.ERROR, ID, NLS.bind(Messages.EngineActivator_1, extension)));
+							}
+						}
+						continue;
+					}
+					File[] listFiles = extension.listFiles();
+					// new magic - multiple info files, f.e.
+					//   egit.info (git feature)
+					//   cdt.link (properties file containing link=path) to other info file
+					for (File file : listFiles) {
+						//if it is a info file - load it
+						if (file.getName().endsWith(".info")) { //$NON-NLS-1$
+							result.add(extension);
+						}
+						// if it is a link - dereference it
+					}
+				} else {
+					synchronized (reportedExtensions) {
+						if (!reportedExtensions.contains(extension)) {
+							reportedExtensions.add(extension);
+							LogHelper.log(new Status(IStatus.WARNING, ID, NLS.bind(Messages.EngineActivator_3, extension)));
+						}
+					}
+				}
+			}
+		}
+		return result;
+	}
 }
diff --git a/bundles/org.eclipse.equinox.p2.engine/src/org/eclipse/equinox/internal/p2/engine/Messages.java b/bundles/org.eclipse.equinox.p2.engine/src/org/eclipse/equinox/internal/p2/engine/Messages.java
index 0b8d79429..c3e0486a2 100644
--- a/bundles/org.eclipse.equinox.p2.engine/src/org/eclipse/equinox/internal/p2/engine/Messages.java
+++ b/bundles/org.eclipse.equinox.p2.engine/src/org/eclipse/equinox/internal/p2/engine/Messages.java
@@ -36,6 +36,12 @@ public class Messages extends NLS {
 	public static String download_artifact;
 	public static String download_no_repository;
 	public static String Engine_Operation_Canceled_By_User;
+
+	public static String EngineActivator_0;
+
+	public static String EngineActivator_1;
+
+	public static String EngineActivator_3;
 	public static String error_parsing_profile;
 	public static String error_persisting_profile;
 	public static String forced_action_execute_error;
@@ -107,6 +113,8 @@ public class Messages extends NLS {
 	public static String SimpleProfileRegistry_States_Error_Writing_File;
 	public static String SimpleProfileRegistry_state_not_found;
 
+	public static String SurrogateProfileHandler_1;
+
 	public static String thread_not_owner;
 	public static String touchpoint_commit_error;
 	public static String touchpoint_prepare_error;
diff --git a/bundles/org.eclipse.equinox.p2.engine/src/org/eclipse/equinox/internal/p2/engine/SimpleProfileRegistry.java b/bundles/org.eclipse.equinox.p2.engine/src/org/eclipse/equinox/internal/p2/engine/SimpleProfileRegistry.java
index 8d55e005f..b5e23013a 100644
--- a/bundles/org.eclipse.equinox.p2.engine/src/org/eclipse/equinox/internal/p2/engine/SimpleProfileRegistry.java
+++ b/bundles/org.eclipse.equinox.p2.engine/src/org/eclipse/equinox/internal/p2/engine/SimpleProfileRegistry.java
@@ -7,6 +7,7 @@
  * Contributors: 
  *     IBM Corporation - initial API and implementation
  *     Ericsson AB - ongoing development
+ *     Red Hat, Inc. - fragments support added.
  ******************************************************************************/
 package org.eclipse.equinox.internal.p2.engine;
 
@@ -262,6 +263,8 @@ public class SimpleProfileRegistry implements IProfileRegistry, IAgentService {
 					//Now that we created a new profile. Tag it, override the property and register the timestamp in the agent registry for pickup by other  
 					internalSetProfileStateProperty(profile, profile.getTimestamp(), IProfile.STATE_PROP_SHARED_INSTALL, IProfile.STATE_SHARED_INSTALL_VALUE_NEW);
 					internalSetProfileStateProperty(profile, profile.getTimestamp(), SIMPLE_PROFILE_REGISTRY_INTERNAL + getBaseTimestamp(profile.getProfileId()), getBaseTimestamp(id));
+					//fragments support - remeber the property
+					internalSetProfileStateProperty(profile, profile.getTimestamp(), SIMPLE_PROFILE_REGISTRY_INTERNAL + getExtTimeStamp(), getExtTimeStamp());
 					agent.registerService(SERVICE_SHARED_INSTALL_NEW_TIMESTAMP, Long.toString(profile.getTimestamp()));
 				} else {
 					//This is the first time we create the shared profile. Tag it as such and also remember the timestamp of the base
@@ -269,21 +272,45 @@ public class SimpleProfileRegistry implements IProfileRegistry, IAgentService {
 					String baseTimestamp = getBaseTimestamp(id);
 					if (baseTimestamp != null)
 						internalSetProfileStateProperty(profile, profile.getTimestamp(), SIMPLE_PROFILE_REGISTRY_INTERNAL + baseTimestamp, baseTimestamp);
+					String extTimestamp = getExtTimeStamp();
+					internalSetProfileStateProperty(profile, profile.getTimestamp(), SIMPLE_PROFILE_REGISTRY_INTERNAL + extTimestamp, extTimestamp);
 				}
 			}
 		}
 		return profile;
 	}
 
+	// get timestamp of fragments (extensions)
+	private String getExtTimeStamp() {
+		long result = -1;
+		if (!EngineActivator.EXTENDED) {
+			return Long.toString(result);
+		}
+		File[] extensions = EngineActivator.getExtensionsDirectories();
+		for (File extension : extensions) {
+			if (extension.lastModified() > result) {
+				result = extension.lastModified();
+			}
+		}
+		return Long.toString(result);
+	}
+
 	private boolean ignoreExistingProfile(IProfile profile) {
 		if (agent.getService(SERVICE_SHARED_INSTALL_NEW_TIMESTAMP) != null)
 			return false;
 
 		String baseTimestamp = getBaseTimestamp(profile.getProfileId());
-		if (baseTimestamp == null)
+		String extTimestamp = getExtTimeStamp();
+		if (baseTimestamp == null) {
 			return false;
+		}
+
+		boolean extensionOK = true;
+		if (surrogateProfileHandler != null && surrogateProfileHandler.isSurrogate(profile)) {
+			extensionOK = (internalGetProfileStateProperties(profile, SIMPLE_PROFILE_REGISTRY_INTERNAL + extTimestamp, false).size() != 0);
+		}
 
-		if (internalGetProfileStateProperties(profile, SIMPLE_PROFILE_REGISTRY_INTERNAL + baseTimestamp, false).size() != 0)
+		if ((internalGetProfileStateProperties(profile, SIMPLE_PROFILE_REGISTRY_INTERNAL + baseTimestamp, false).size() != 0) && extensionOK)
 			return false;
 
 		return true;
@@ -671,6 +698,10 @@ public class SimpleProfileRegistry implements IProfileRegistry, IAgentService {
 	class Parser extends ProfileParser {
 		private final Map<String, ProfileHandler> profileHandlers = new HashMap<String, ProfileHandler>();
 
+		public Map<String, ProfileHandler> getProfileHandlers() {
+			return Collections.unmodifiableMap(profileHandlers);
+		}
+
 		public Parser(BundleContext context, String bundleId) {
 			super(context, bundleId);
 		}
diff --git a/bundles/org.eclipse.equinox.p2.engine/src/org/eclipse/equinox/internal/p2/engine/SurrogateProfileHandler.java b/bundles/org.eclipse.equinox.p2.engine/src/org/eclipse/equinox/internal/p2/engine/SurrogateProfileHandler.java
index e99a0f2c1..666f5d85c 100644
--- a/bundles/org.eclipse.equinox.p2.engine/src/org/eclipse/equinox/internal/p2/engine/SurrogateProfileHandler.java
+++ b/bundles/org.eclipse.equinox.p2.engine/src/org/eclipse/equinox/internal/p2/engine/SurrogateProfileHandler.java
@@ -8,17 +8,19 @@
  *  Contributors:
  *      IBM Corporation - initial API and implementation
  *      Ericsson AB - Bug 400011 - [shared] Cleanup the SurrogateProfileHandler code
+ *      Red Hat, Inc. - fragments support added.
  *******************************************************************************/
 package org.eclipse.equinox.internal.p2.engine;
 
-import java.io.File;
+import java.io.*;
 import java.lang.ref.SoftReference;
 import java.net.*;
-import java.util.ArrayList;
-import java.util.Iterator;
-import org.eclipse.core.runtime.IProgressMonitor;
-import org.eclipse.core.runtime.URIUtil;
+import java.util.*;
+import org.eclipse.core.runtime.*;
+import org.eclipse.equinox.internal.p2.core.helpers.LogHelper;
 import org.eclipse.equinox.internal.p2.core.helpers.ServiceHelper;
+import org.eclipse.equinox.internal.p2.engine.ProfileParser.ProfileHandler;
+import org.eclipse.equinox.internal.p2.engine.SimpleProfileRegistry.Parser;
 import org.eclipse.equinox.p2.core.IProvisioningAgent;
 import org.eclipse.equinox.p2.engine.IProfile;
 import org.eclipse.equinox.p2.metadata.*;
@@ -26,6 +28,9 @@ import org.eclipse.equinox.p2.metadata.MetadataFactory.InstallableUnitDescriptio
 import org.eclipse.equinox.p2.metadata.expression.ExpressionUtil;
 import org.eclipse.equinox.p2.metadata.expression.IMatchExpression;
 import org.eclipse.equinox.p2.query.*;
+import org.eclipse.equinox.p2.repository.IRepositoryManager;
+import org.eclipse.equinox.p2.repository.artifact.IArtifactRepositoryManager;
+import org.eclipse.equinox.p2.repository.metadata.IMetadataRepositoryManager;
 import org.eclipse.osgi.service.datalocation.Location;
 import org.eclipse.osgi.util.NLS;
 
@@ -156,25 +161,112 @@ public class SurrogateProfileHandler implements ISurrogateProfileHandler {
 				return profile;
 		}
 
-		final IProfile profile = registry.getProfile(id, currentTimestamp);
+		final Profile profile = (Profile) registry.getProfile(id, currentTimestamp);
 		if (profile != null)
 			cachedProfile = new SoftReference<IProfile>(profile);
+
+		if (!EngineActivator.EXTENDED) {
+			return profile;
+		}
+
+		setUpRepos();
 		return profile;
 	}
 
+	/**
+	 * Removes repositories from fragments locations as they might be obsolete and adds them back.
+	 */
+	private void setUpRepos() {
+		//clean old junk
+		IMetadataRepositoryManager metaManager = (IMetadataRepositoryManager) agent.getService(IMetadataRepositoryManager.SERVICE_NAME);
+		URI[] knownRepositories = metaManager.getKnownRepositories(IRepositoryManager.REPOSITORIES_LOCAL);
+		for (URI uri : knownRepositories) {
+			if ("true".equals(metaManager.getRepositoryProperty(uri, EngineActivator.P2_FRAGMENT_PROPERTY))) { //$NON-NLS-1$
+				metaManager.removeRepository(uri);
+			}
+		}
+
+		IArtifactRepositoryManager artifactManager = (IArtifactRepositoryManager) agent.getService(IArtifactRepositoryManager.SERVICE_NAME);
+		knownRepositories = artifactManager.getKnownRepositories(IRepositoryManager.REPOSITORIES_LOCAL);
+		for (URI uri : knownRepositories) {
+			if ("true".equals(artifactManager.getRepositoryProperty(uri, EngineActivator.P2_FRAGMENT_PROPERTY))) { //$NON-NLS-1$
+				artifactManager.removeRepository(uri);
+			}
+		}
+
+		File[] fragments = EngineActivator.getExtensionsDirectories();
+		for (File f : fragments) {
+			metaManager.addRepository(f.toURI());
+			metaManager.setRepositoryProperty(f.toURI(), EngineActivator.P2_FRAGMENT_PROPERTY, Boolean.TRUE.toString());
+			artifactManager.addRepository(f.toURI());
+			artifactManager.setRepositoryProperty(f.toURI(), EngineActivator.P2_FRAGMENT_PROPERTY, Boolean.TRUE.toString());
+		}
+	}
+
 	/* (non-Javadoc)
 	 * @see org.eclipse.equinox.internal.p2.engine.ISurrogateProfileHandler#createProfile(java.lang.String)
 	 */
 	public IProfile createProfile(String id) {
-		final IProfile sharedProfile = getSharedProfile(id);
+		final Profile sharedProfile = (Profile) getSharedProfile(id);
 		if (sharedProfile == null)
 			return null;
 
+		if (!EngineActivator.EXTENDED) {
+			Profile userProfile = new Profile(agent, id, null, sharedProfile.getProperties());
+			userProfile.setProperty(PROP_SURROGATE, Boolean.TRUE.toString());
+			userProfile.setSurrogateProfileHandler(this);
+			updateProperties(sharedProfile, userProfile);
+			addSharedProfileBaseIUs(sharedProfile, userProfile);
+
+			return userProfile;
+		}
+
+		File[] extensionLocations = EngineActivator.getExtensionsDirectories();
+		Set<IInstallableUnit> added = new HashSet<IInstallableUnit>();
+		for (File extension : extensionLocations) {
+			if (extension.isDirectory()) {
+				File[] listFiles = extension.listFiles(new FileFilter() {
+
+					public boolean accept(File pathname) {
+						if (pathname.getName().endsWith(".profile")) { //$NON-NLS-1$
+							return true;
+						}
+						return false;
+					}
+				});
+				for (File profileFile : listFiles) {
+					Parser extensionParser = profileRegistry.new Parser(EngineActivator.getContext(), EngineActivator.ID);
+					try {
+						extensionParser.parse(profileFile);
+						//there is only one profile as we read only one
+						String key = extensionParser.getProfileHandlers().keySet().iterator().next();
+
+						ProfileHandler extensionHandler = extensionParser.getProfileHandlers().get(key);
+						IInstallableUnit[] installableUnits = extensionHandler.getInstallableUnits();
+						for (IInstallableUnit unit : installableUnits) {
+							if (!added.contains(unit)) {
+								added.add(unit);
+								sharedProfile.addInstallableUnit(unit);
+							}
+							Map<String, String> iuProperties = extensionHandler.getIUProperties(unit);
+							if (iuProperties != null && !iuProperties.isEmpty()) {
+								sharedProfile.addInstallableUnitProperties(unit, iuProperties);
+							}
+						}
+					} catch (IOException e) {
+						LogHelper.log(new Status(IStatus.ERROR, EngineActivator.ID, NLS.bind(Messages.SurrogateProfileHandler_1, profileFile), e));
+					}
+				}
+				continue;
+			}
+		}
+
 		Profile userProfile = new Profile(agent, id, null, sharedProfile.getProperties());
 		userProfile.setProperty(PROP_SURROGATE, Boolean.TRUE.toString());
 		userProfile.setSurrogateProfileHandler(this);
 		updateProperties(sharedProfile, userProfile);
 		addSharedProfileBaseIUs(sharedProfile, userProfile);
+
 		return userProfile;
 	}
 
diff --git a/bundles/org.eclipse.equinox.p2.engine/src/org/eclipse/equinox/internal/p2/engine/messages.properties b/bundles/org.eclipse.equinox.p2.engine/src/org/eclipse/equinox/internal/p2/engine/messages.properties
index f5af6f295..68f2d861b 100644
--- a/bundles/org.eclipse.equinox.p2.engine/src/org/eclipse/equinox/internal/p2/engine/messages.properties
+++ b/bundles/org.eclipse.equinox.p2.engine/src/org/eclipse/equinox/internal/p2/engine/messages.properties
@@ -52,6 +52,7 @@ reg_dir_not_available=Registry Directory not available: {0}.
 thread_not_owner=Thread not lock owner.
 profile_lock_not_reentrant=Lock failed. Profile does not permit reentrant locking.
 Shared_Profile=Shared profile
+SurrogateProfileHandler_1=Error while parsing extension profile {0}
 shared_profile_not_found=Shared profile {0} not found.
 
 ProfilePreferences_saving=Saving profile preferences
@@ -90,6 +91,9 @@ rollingback_error=An error was detected while performing the engine operation an
 rollingback_cancel=The engine operation was cancelled and the changes are being rolled back.
 
 Engine_Operation_Canceled_By_User=Operation canceled by the user.
+EngineActivator_0=Error while loading repository from {0}
+EngineActivator_1=Fragment directory {0} is not read only and will not be scanned for fragments.
+EngineActivator_3=No idea what to do with {0}. It does not contain valid fragment.
 InstallableUnitEvent_type_not_install_or_uninstall_or_configure=type must be either UNINSTALL(0) or INSTALL(1) or UNCONFIGURE(2) or CONFIGURE(3)
 CertificateChecker_CertificateError=An invalid certificate was found.
 CertificateChecker_CertificateRejected=One or more certificates rejected. Cannot proceed with installation.