Bug 464614 Use XZ as compression formats of metadata filesI20150514-2000I20150514-1000I20150513-2000

This part takes care of the artifacts.xml file.

Change-Id: If18777d4057ec273d219c4aa2b959fa3093a9a7f

17 files changed, 251 insertions, 4 deletions

diff --git a/bundles/org.eclipse.equinox.p2.artifact.repository/META-INF/MANIFEST.MF b/bundles/org.eclipse.equinox.p2.artifact.repository/META-INF/MANIFEST.MF
index 1e5d88f95..40a119189 100644
--- a/bundles/org.eclipse.equinox.p2.artifact.repository/META-INF/MANIFEST.MF
+++ b/bundles/org.eclipse.equinox.p2.artifact.repository/META-INF/MANIFEST.MF
@@ -33,7 +33,8 @@ Export-Package: org.eclipse.equinox.internal.p2.artifact.processing;x-friends:="
    org.eclipse.equinox.p2.updatesite"
 Require-Bundle: org.eclipse.equinox.common;bundle-version="[3.7.0,4.0.0)",
  org.eclipse.equinox.registry,
- org.eclipse.equinox.p2.metadata
+ org.eclipse.equinox.p2.metadata,
+ org.tukaani.xz;bundle-version="1.3.0"
 Import-Package: javax.xml.parsers,
  org.eclipse.core.runtime.jobs,
  org.eclipse.equinox.internal.p2.core.helpers,
diff --git a/bundles/org.eclipse.equinox.p2.artifact.repository/plugin.xml b/bundles/org.eclipse.equinox.p2.artifact.repository/plugin.xml
index 7025c7f08..fa791fe2c 100644
--- a/bundles/org.eclipse.equinox.p2.artifact.repository/plugin.xml
+++ b/bundles/org.eclipse.equinox.p2.artifact.repository/plugin.xml
@@ -11,7 +11,12 @@
 		<step class="org.eclipse.equinox.internal.p2.artifact.processors.pack200.Pack200ProcessorStep"/>
 	</extension>
 
-   <extension id="simpleRepository"  point="org.eclipse.equinox.p2.artifact.repository.artifactRepositories">
+   <extension id="XZedRepository"  point="org.eclipse.equinox.p2.artifact.repository.artifactRepositories">
+		<filter suffix="artifacts.xml.xz"/>
+		<factory class="org.eclipse.equinox.internal.p2.artifact.repository.simple.XZedSimpleArtifactRepositoryFactory"/>
+   </extension>
+   
+    <extension id="simpleRepository"  point="org.eclipse.equinox.p2.artifact.repository.artifactRepositories">
 		<filter suffix="artifacts.xml"/>
 		<factory class="org.eclipse.equinox.internal.p2.artifact.repository.simple.SimpleArtifactRepositoryFactory"/>
    </extension>
diff --git a/bundles/org.eclipse.equinox.p2.artifact.repository/src/org/eclipse/equinox/internal/p2/artifact/repository/simple/XZedSimpleArtifactRepositoryFactory.java b/bundles/org.eclipse.equinox.p2.artifact.repository/src/org/eclipse/equinox/internal/p2/artifact/repository/simple/XZedSimpleArtifactRepositoryFactory.java
new file mode 100644
index 000000000..2ed52fd10
--- /dev/null
+++ b/bundles/org.eclipse.equinox.p2.artifact.repository/src/org/eclipse/equinox/internal/p2/artifact/repository/simple/XZedSimpleArtifactRepositoryFactory.java
@@ -0,0 +1,125 @@
+/*******************************************************************************
+ * Copyright (c) 2015  Rapicorp, Inc and others.
+ * All rights reserved. This program and the accompanying materials
+ * are made available under the terms of the Eclipse Public License v1.0
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/epl-v10.html
+ *
+ * Contributors:
+ *     Rapicorp, Inc - initial API and implementation
+ *******************************************************************************/
+package org.eclipse.equinox.internal.p2.artifact.repository.simple;
+
+import java.io.*;
+import java.net.URI;
+import java.util.Map;
+import org.eclipse.core.runtime.*;
+import org.eclipse.equinox.internal.p2.artifact.repository.Activator;
+import org.eclipse.equinox.internal.p2.artifact.repository.Messages;
+import org.eclipse.equinox.internal.p2.core.helpers.Tracing;
+import org.eclipse.equinox.internal.p2.repository.CacheManager;
+import org.eclipse.equinox.p2.core.ProvisionException;
+import org.eclipse.equinox.p2.repository.IRepositoryManager;
+import org.eclipse.equinox.p2.repository.artifact.IArtifactRepository;
+import org.eclipse.equinox.p2.repository.artifact.spi.ArtifactRepositoryFactory;
+import org.eclipse.osgi.util.NLS;
+import org.tukaani.xz.XZInputStream;
+
+public class XZedSimpleArtifactRepositoryFactory extends ArtifactRepositoryFactory {
+	private static final String REPOSITORY_FILENAME = "artifacts.xml.xz"; //$NON-NLS-1$
+	private static final String PROTOCOL_FILE = "file"; //$NON-NLS-1$
+
+	public IArtifactRepository create(URI location, String name, String type, Map<String, String> properties) {
+		return new SimpleArtifactRepository(getAgent(), name, location, properties);
+	}
+
+	public IArtifactRepository load(URI location, int flags, IProgressMonitor monitor) throws ProvisionException {
+		return load(location, flags, monitor, true);
+	}
+
+	/*
+	 * Returns a file in the local file system that contains the contents of the
+	 * metadata repository at the given location.
+	 */
+	private File getLocalFile(URI location, IProgressMonitor monitor) throws IOException, ProvisionException {
+		File localFile = null;
+		URI xzLocation = URIUtil.append(location, REPOSITORY_FILENAME);
+		// If the repository is local, we can return the repository file directly
+		if (PROTOCOL_FILE.equals(xzLocation.getScheme())) {
+			//look for a compressed local file
+			localFile = URIUtil.toFile(xzLocation);
+			if (localFile.exists())
+				return localFile;
+			String msg = NLS.bind(Messages.io_failedRead, location);
+			throw new ProvisionException(new Status(IStatus.ERROR, Activator.ID, ProvisionException.REPOSITORY_NOT_FOUND, msg, null));
+		}
+		// file is not local, create a cache of the repository metadata
+		CacheManager cache = (CacheManager) getAgent().getService(CacheManager.SERVICE_NAME);
+		if (cache == null)
+			throw new IllegalArgumentException("Cache manager service not available"); //$NON-NLS-1$
+		localFile = cache.createCacheFromFile(URIUtil.append(location, REPOSITORY_FILENAME), monitor);
+		if (localFile == null) {
+			// there is no remote file in either form - this should not really happen as
+			// createCache should bail out with exception if something is wrong. This is an internal
+			// error.
+			throw new ProvisionException(new Status(IStatus.ERROR, Activator.ID, ProvisionException.REPOSITORY_NOT_FOUND, Messages.repoMan_internalError, null));
+		}
+		return localFile;
+	}
+
+	/* (non-Javadoc)
+	 * @see org.eclipse.equinox.p2.repository.metadata.spi.MetadataRepositoryFactory#load(java.net.URL, org.eclipse.core.runtime.IProgressMonitor)
+	 */
+	public IArtifactRepository load(URI location, int flags, IProgressMonitor monitor, boolean acquireLock) throws ProvisionException {
+		long time = 0;
+		final String debugMsg = "Restoring artifact repository "; //$NON-NLS-1$
+		if (Tracing.DEBUG_METADATA_PARSING) {
+			Tracing.debug(debugMsg + location);
+			time = -System.currentTimeMillis();
+		}
+		SubMonitor sub = SubMonitor.convert(monitor, 400);
+		try {
+			File localFile = getLocalFile(location, sub.newChild(300));
+			InputStream stream = new BufferedInputStream(new FileInputStream(localFile));
+			XZInputStream descriptorStream = new XZInputStream(stream);
+			try {
+				//parse the repository descriptor file
+				sub.setWorkRemaining(100);
+				SimpleArtifactRepository result = (SimpleArtifactRepository) new SimpleArtifactRepositoryIO(getAgent()).read(localFile.toURI(), descriptorStream, sub.newChild(100), acquireLock);
+				if (result != null && (flags & IRepositoryManager.REPOSITORY_HINT_MODIFIABLE) > 0 && !result.isModifiable())
+					return null;
+				result.initializeAfterLoad(location);
+				if (Tracing.DEBUG_METADATA_PARSING) {
+					time += System.currentTimeMillis();
+					Tracing.debug(debugMsg + "time (ms): " + time); //$NON-NLS-1$ 
+				}
+				return result;
+			} finally {
+				safeClose(descriptorStream);
+				safeClose(stream);
+			}
+		} catch (FileNotFoundException e) {
+			String msg = NLS.bind(Messages.io_failedRead, location);
+			throw new ProvisionException(new Status(IStatus.ERROR, Activator.ID, ProvisionException.REPOSITORY_NOT_FOUND, msg, e));
+		} catch (IOException e) {
+			String msg = NLS.bind(Messages.io_failedRead, location);
+			throw new ProvisionException(new Status(IStatus.ERROR, Activator.ID, ProvisionException.REPOSITORY_FAILED_READ, msg, e));
+		} finally {
+			if (monitor != null)
+				monitor.done();
+		}
+	}
+
+	/**
+	 * Closes a stream, ignoring any secondary exceptions
+	 */
+	private void safeClose(InputStream stream) {
+		if (stream == null)
+			return;
+		try {
+			stream.close();
+		} catch (IOException e) {
+			//ignore
+		}
+	}
+}
diff --git a/bundles/org.eclipse.equinox.p2.tests/src/org/eclipse/equinox/p2/tests/metadata/repository/XZedRepositoryTest.java b/bundles/org.eclipse.equinox.p2.tests/src/org/eclipse/equinox/p2/tests/metadata/repository/XZedRepositoryTest.java
index 7f55df4e1..3042424c0 100644
--- a/bundles/org.eclipse.equinox.p2.tests/src/org/eclipse/equinox/p2/tests/metadata/repository/XZedRepositoryTest.java
+++ b/bundles/org.eclipse.equinox.p2.tests/src/org/eclipse/equinox/p2/tests/metadata/repository/XZedRepositoryTest.java
@@ -12,10 +12,11 @@ package org.eclipse.equinox.p2.tests.metadata.repository;
 
 import org.eclipse.core.runtime.OperationCanceledException;
 import org.eclipse.equinox.p2.core.ProvisionException;
-import org.eclipse.equinox.p2.metadata.IInstallableUnit;
-import org.eclipse.equinox.p2.metadata.Version;
+import org.eclipse.equinox.p2.metadata.*;
 import org.eclipse.equinox.p2.query.IQueryResult;
 import org.eclipse.equinox.p2.query.QueryUtil;
+import org.eclipse.equinox.p2.repository.artifact.ArtifactKeyQuery;
+import org.eclipse.equinox.p2.repository.artifact.IArtifactRepository;
 import org.eclipse.equinox.p2.repository.metadata.IMetadataRepository;
 import org.eclipse.equinox.p2.tests.AbstractProvisioningTest;
 import org.junit.Test;
@@ -61,4 +62,46 @@ public class XZedRepositoryTest extends AbstractProvisioningTest {
 		assertEquals(1, units.toSet().size());
 	}
 
+	@Test
+	public void testArtifactsJarAndXZ() throws ProvisionException, OperationCanceledException {
+		IArtifactRepository repo = getArtifactRepositoryManager().loadRepository(getTestData("xzedRepo", "testData/xzRepoTests/artifacts/artifactsJarAndXZ").toURI(), null);
+		IQueryResult<IArtifactKey> units = repo.query(new ArtifactKeyQuery("osgi.bundle", "aaPlugin", new VersionRange("[1.0.0, 1.0.0]")), null);
+
+		assertEquals(1, units.toSet().size());
+	}
+
+	@Test
+	public void testxzAndArtifactsJar() throws ProvisionException, OperationCanceledException {
+		IArtifactRepository repo = getArtifactRepositoryManager().loadRepository(getTestData("xzedRepo", "testData/xzRepoTests/artifacts/xzAndArtifactsJar").toURI(), null);
+		IQueryResult<IArtifactKey> units = repo.query(new ArtifactKeyQuery("osgi.bundle", "aaPluginFromXZ", new VersionRange("[1.0.0, 1.0.0]")), null);
+
+		assertEquals(1, units.toSet().size());
+	}
+
+	@Test
+	public void testxzAndArtifactsXML() throws ProvisionException, OperationCanceledException {
+		IArtifactRepository repo = getArtifactRepositoryManager().loadRepository(getTestData("xzedRepo", "testData/xzRepoTests/artifacts/xzAndArtifactsXML").toURI(), null);
+		IQueryResult<IArtifactKey> units = repo.query(new ArtifactKeyQuery("osgi.bundle", "aaPluginFromXZ", new VersionRange("[1.0.0, 1.0.0]")), null);
+
+		assertEquals(1, units.toSet().size());
+	}
+
+	@Test
+	public void testxzOnly() throws ProvisionException, OperationCanceledException {
+		IArtifactRepository repo = getArtifactRepositoryManager().loadRepository(getTestData("xzedRepo", "testData/xzRepoTests/artifacts/xzOnly").toURI(), null);
+		IQueryResult<IArtifactKey> units = repo.query(new ArtifactKeyQuery("osgi.bundle", "aaPluginFromXZ", new VersionRange("[1.0.0, 1.0.0]")), null);
+
+		assertEquals(1, units.toSet().size());
+	}
+
+	@Test
+	public void testLoadArtifactsXzBusted() throws ProvisionException, OperationCanceledException {
+		boolean repoCanLoad = true;
+		try {
+			getMetadataRepositoryManager().loadRepository(getTestData("xzedRepo", "testData/xzRepoTests/artifacts/xzBusted").toURI(), null);
+		} catch (ProvisionException e) {
+			repoCanLoad = false;
+		}
+		assertFalse(repoCanLoad);
+	}
 }
diff --git a/bundles/org.eclipse.equinox.p2.tests/testData/xzRepoTests/artifacts/artifactsJarAndXZ/artifacts.jar b/bundles/org.eclipse.equinox.p2.tests/testData/xzRepoTests/artifacts/artifactsJarAndXZ/artifacts.jar
new file mode 100644
index 000000000..f510f4fdf
--- /dev/null
+++ b/bundles/org.eclipse.equinox.p2.tests/testData/xzRepoTests/artifacts/artifactsJarAndXZ/artifacts.jar
diff --git a/bundles/org.eclipse.equinox.p2.tests/testData/xzRepoTests/artifacts/artifactsJarAndXZ/artifacts.xml.xz b/bundles/org.eclipse.equinox.p2.tests/testData/xzRepoTests/artifacts/artifactsJarAndXZ/artifacts.xml.xz
new file mode 100644
index 000000000..ebd5b158e
--- /dev/null
+++ b/bundles/org.eclipse.equinox.p2.tests/testData/xzRepoTests/artifacts/artifactsJarAndXZ/artifacts.xml.xz
diff --git a/bundles/org.eclipse.equinox.p2.tests/testData/xzRepoTests/artifacts/artifactsJarAndXZ/p2.index b/bundles/org.eclipse.equinox.p2.tests/testData/xzRepoTests/artifacts/artifactsJarAndXZ/p2.index
new file mode 100644
index 000000000..839913f17
--- /dev/null
+++ b/bundles/org.eclipse.equinox.p2.tests/testData/xzRepoTests/artifacts/artifactsJarAndXZ/p2.index
@@ -0,0 +1,3 @@
+artifact.repository.factory.order= artifacts.xml,artifacts.xml.xz,!
+version=1
+
diff --git a/bundles/org.eclipse.equinox.p2.tests/testData/xzRepoTests/artifacts/xzAndArtifactsJar/artifacts.jar b/bundles/org.eclipse.equinox.p2.tests/testData/xzRepoTests/artifacts/xzAndArtifactsJar/artifacts.jar
new file mode 100644
index 000000000..f510f4fdf
--- /dev/null
+++ b/bundles/org.eclipse.equinox.p2.tests/testData/xzRepoTests/artifacts/xzAndArtifactsJar/artifacts.jar
diff --git a/bundles/org.eclipse.equinox.p2.tests/testData/xzRepoTests/artifacts/xzAndArtifactsJar/artifacts.xml.xz b/bundles/org.eclipse.equinox.p2.tests/testData/xzRepoTests/artifacts/xzAndArtifactsJar/artifacts.xml.xz
new file mode 100644
index 000000000..ebd5b158e
--- /dev/null
+++ b/bundles/org.eclipse.equinox.p2.tests/testData/xzRepoTests/artifacts/xzAndArtifactsJar/artifacts.xml.xz
diff --git a/bundles/org.eclipse.equinox.p2.tests/testData/xzRepoTests/artifacts/xzAndArtifactsJar/p2.index b/bundles/org.eclipse.equinox.p2.tests/testData/xzRepoTests/artifacts/xzAndArtifactsJar/p2.index
new file mode 100644
index 000000000..eba465705
--- /dev/null
+++ b/bundles/org.eclipse.equinox.p2.tests/testData/xzRepoTests/artifacts/xzAndArtifactsJar/p2.index
@@ -0,0 +1,3 @@
+artifact.repository.factory.order= artifacts.xml.xz,artifacts.xml,!
+version=1
+
diff --git a/bundles/org.eclipse.equinox.p2.tests/testData/xzRepoTests/artifacts/xzAndArtifactsXML/artifacts.xml b/bundles/org.eclipse.equinox.p2.tests/testData/xzRepoTests/artifacts/xzAndArtifactsXML/artifacts.xml
new file mode 100644
index 000000000..19e55d746
--- /dev/null
+++ b/bundles/org.eclipse.equinox.p2.tests/testData/xzRepoTests/artifacts/xzAndArtifactsXML/artifacts.xml
@@ -0,0 +1,29 @@
+<?xml version='1.0' encoding='UTF-8'?>
+<?artifactRepository class='org.eclipse.equinox.internal.p2.artifact.repository.simple.SimpleArtifactRepository' version='1.0.0'?>
+<repository name='Good Test Repository' type='org.eclipse.equinox.p2.artifact.repository.simpleRepository' version='1' description='Good test repository description'>
+  <properties size='3'>
+    <property name='p2.compressed' value='false'/>
+    <property name='p2.timestamp' value='1218734853468'/>
+    <property name='p2.system' value='false'/>
+  </properties>
+  <mappings size='3'>
+    <rule filter='(&amp; (classifier=osgi.bundle))' output='${repoUrl}/plugins/${id}_${version}.jar'/>
+    <rule filter='(&amp; (classifier=binary))' output='${repoUrl}/binary/${id}_${version}'/>
+    <rule filter='(&amp; (classifier=org.eclipse.update.feature))' output='${repoUrl}/features/${id}_${version}.jar'/>
+  </mappings>
+  <artifacts size='2'>
+    <artifact classifier='osgi.bundle' id='aaPlugin' version='1.0.0'>
+      <properties size='3'>
+        <property name='artifact.size' value='469'/>
+        <property name='download.size' value='469'/>
+        <property name='download.contentType' value='application/zip'/>
+      </properties>
+    </artifact>
+    <artifact classifier='org.eclipse.update.feature' id='aaFeature' version='1.0.0'>
+      <properties size='2'>
+        <property name='artifact.size' value='670'/>
+        <property name='download.size' value='670'/>
+      </properties>
+    </artifact>
+  </artifacts>
+</repository>
diff --git a/bundles/org.eclipse.equinox.p2.tests/testData/xzRepoTests/artifacts/xzAndArtifactsXML/artifacts.xml.xz b/bundles/org.eclipse.equinox.p2.tests/testData/xzRepoTests/artifacts/xzAndArtifactsXML/artifacts.xml.xz
new file mode 100644
index 000000000..ebd5b158e
--- /dev/null
+++ b/bundles/org.eclipse.equinox.p2.tests/testData/xzRepoTests/artifacts/xzAndArtifactsXML/artifacts.xml.xz
diff --git a/bundles/org.eclipse.equinox.p2.tests/testData/xzRepoTests/artifacts/xzAndArtifactsXML/p2.index b/bundles/org.eclipse.equinox.p2.tests/testData/xzRepoTests/artifacts/xzAndArtifactsXML/p2.index
new file mode 100644
index 000000000..eba465705
--- /dev/null
+++ b/bundles/org.eclipse.equinox.p2.tests/testData/xzRepoTests/artifacts/xzAndArtifactsXML/p2.index
@@ -0,0 +1,3 @@
+artifact.repository.factory.order= artifacts.xml.xz,artifacts.xml,!
+version=1
+
diff --git a/bundles/org.eclipse.equinox.p2.tests/testData/xzRepoTests/artifacts/xzBusted/artifacts.xml.xz b/bundles/org.eclipse.equinox.p2.tests/testData/xzRepoTests/artifacts/xzBusted/artifacts.xml.xz
new file mode 100644
index 000000000..19e55d746
--- /dev/null
+++ b/bundles/org.eclipse.equinox.p2.tests/testData/xzRepoTests/artifacts/xzBusted/artifacts.xml.xz
@@ -0,0 +1,29 @@
+<?xml version='1.0' encoding='UTF-8'?>
+<?artifactRepository class='org.eclipse.equinox.internal.p2.artifact.repository.simple.SimpleArtifactRepository' version='1.0.0'?>
+<repository name='Good Test Repository' type='org.eclipse.equinox.p2.artifact.repository.simpleRepository' version='1' description='Good test repository description'>
+  <properties size='3'>
+    <property name='p2.compressed' value='false'/>
+    <property name='p2.timestamp' value='1218734853468'/>
+    <property name='p2.system' value='false'/>
+  </properties>
+  <mappings size='3'>
+    <rule filter='(&amp; (classifier=osgi.bundle))' output='${repoUrl}/plugins/${id}_${version}.jar'/>
+    <rule filter='(&amp; (classifier=binary))' output='${repoUrl}/binary/${id}_${version}'/>
+    <rule filter='(&amp; (classifier=org.eclipse.update.feature))' output='${repoUrl}/features/${id}_${version}.jar'/>
+  </mappings>
+  <artifacts size='2'>
+    <artifact classifier='osgi.bundle' id='aaPlugin' version='1.0.0'>
+      <properties size='3'>
+        <property name='artifact.size' value='469'/>
+        <property name='download.size' value='469'/>
+        <property name='download.contentType' value='application/zip'/>
+      </properties>
+    </artifact>
+    <artifact classifier='org.eclipse.update.feature' id='aaFeature' version='1.0.0'>
+      <properties size='2'>
+        <property name='artifact.size' value='670'/>
+        <property name='download.size' value='670'/>
+      </properties>
+    </artifact>
+  </artifacts>
+</repository>
diff --git a/bundles/org.eclipse.equinox.p2.tests/testData/xzRepoTests/artifacts/xzBusted/p2.index b/bundles/org.eclipse.equinox.p2.tests/testData/xzRepoTests/artifacts/xzBusted/p2.index
new file mode 100644
index 000000000..0c59cd1de
--- /dev/null
+++ b/bundles/org.eclipse.equinox.p2.tests/testData/xzRepoTests/artifacts/xzBusted/p2.index
@@ -0,0 +1,3 @@
+artifact.repository.factory.order= artifacts.xml.xz,!
+version=1
+
diff --git a/bundles/org.eclipse.equinox.p2.tests/testData/xzRepoTests/artifacts/xzOnly/artifacts.xml.xz b/bundles/org.eclipse.equinox.p2.tests/testData/xzRepoTests/artifacts/xzOnly/artifacts.xml.xz
new file mode 100644
index 000000000..ebd5b158e
--- /dev/null
+++ b/bundles/org.eclipse.equinox.p2.tests/testData/xzRepoTests/artifacts/xzOnly/artifacts.xml.xz
diff --git a/bundles/org.eclipse.equinox.p2.tests/testData/xzRepoTests/artifacts/xzOnly/p2.index b/bundles/org.eclipse.equinox.p2.tests/testData/xzRepoTests/artifacts/xzOnly/p2.index
new file mode 100644
index 000000000..0c59cd1de
--- /dev/null
+++ b/bundles/org.eclipse.equinox.p2.tests/testData/xzRepoTests/artifacts/xzOnly/p2.index
@@ -0,0 +1,3 @@
+artifact.repository.factory.order= artifacts.xml.xz,!
+version=1
+