Bug 422366 - use target file to specify repositories and exact version

of non-platform prereqs
(includes some changes to "production" scripts, required to test this in
local test machine ... they all match was is in master, and mentioned in
other releng bugs). 

9 files changed, 115 insertions, 67 deletions

diff --git a/eclipse-platform-parent/pom.xml b/eclipse-platform-parent/pom.xml
index 0fe8355f5..f5bde06f8 100644
--- a/eclipse-platform-parent/pom.xml
+++ b/eclipse-platform-parent/pom.xml
@@ -33,17 +33,16 @@
 
     <comparator.repo>http://download.eclipse.org/eclipse/updates/4.3-M-builds</comparator.repo>
 
-    <jetty-repo.url>http://download.eclipse.org/jetty/updates/jetty-bundles-8.x/8.1.10.v20130312/</jetty-repo.url>
-    <ecf-repo.url>http://download.eclipse.org/rt/ecf/int7/site.p2/</ecf-repo.url>
+    <!-- 
+      NOTE: this EMF URL must "match" exactly what is in the eclipse-sdk-prereqs.target file. 
+      We use it (the EMF repo) not only for resolving pre-reqs during the build, but also in 'repository' POM for mirroring
+      a copy of the EMF features into our own repository. This is required since we merely "require" EMF (not "include" it).
+      And, of course, in the PDE target file, the URL is expected to be the actual URL, not a variable. 
+    -->
     <emf-repo.url>http://download.eclipse.org/modeling/emf/emf/updates/2.9.x/base/S201308270309/</emf-repo.url>
-    <aspectj-repo.url>http://download.eclipse.org/tools/ajdt/aspectj/update/e42</aspectj-repo.url>
-    <egit-repo.url>http://download.eclipse.org/egit/staging/v3.0.0.201305080800-m7</egit-repo.url>
-    
-    <!-- CBI provided common license feature. (bug 394793) -->
-    <license-repo.url>http://download.eclipse.org/cbi/updates/license/</license-repo.url>
-    
+
     <tycho-repo.url>https://oss.sonatype.org/content/repositories/public/</tycho-repo.url>
-   
+
     <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
     <cbi-jdt-repo.url>https://repo.eclipse.org/content/repositories/eclipse-staging/</cbi-jdt-repo.url>
     <cbi-jdt-version>3.9.0.v20130604-1421</cbi-jdt-version>
@@ -52,39 +51,7 @@
     <eclipse-p2-repo.url>http://download.eclipse.org/eclipse/updates/4.3-M-builds</eclipse-p2-repo.url>
 
   </properties>
-  <repositories>
-    <repository>
-      <id>ecf</id>
-      <url>${ecf-repo.url}</url>
-      <layout>p2</layout>
-    </repository>
-    <repository>
-      <id>jetty</id>
-      <url>${jetty-repo.url}</url>
-      <layout>p2</layout>
-    </repository>
-    <repository>
-      <id>emf</id>
-      <url>${emf-repo.url}</url>
-      <layout>p2</layout>
-    </repository>
-    <repository>
-      <id>license-feature</id>
-      <url>${license-repo.url}</url>
-      <layout>p2</layout>
-    </repository>
-    <repository>
-      <id>egit</id>
-      <url>${egit-repo.url}</url>
-      <layout>p2</layout>
-    </repository>
-    <repository>
-      <!-- aspectj is used by some rt.equinox.bundles and is required by sdk-tests feature -->
-      <id>aspectj</id>
-      <url>${aspectj-repo.url}</url>
-      <layout>p2</layout>
-    </repository>
-  </repositories>
+
   <distributionManagement>
     <repository>
       <id>repo.eclipse.org</id>
diff --git a/eclipse-platform-parent/saveproperties/mavenproperties.properties b/eclipse-platform-parent/saveproperties/mavenproperties.properties
index a34d3a2e0..bd73806fb 100644
--- a/eclipse-platform-parent/saveproperties/mavenproperties.properties
+++ b/eclipse-platform-parent/saveproperties/mavenproperties.properties
@@ -32,6 +32,3 @@ eclipse-repo.url=${eclipse-repo.url}
 
 sun.java.command=${sun.java.command}
 
-
-
-
diff --git a/eclipse.platform.releng.prereqs.sdk/eclipse-sdk-prereqs.target b/eclipse.platform.releng.prereqs.sdk/eclipse-sdk-prereqs.target
index 4cbd98d93..c4cb4efb8 100644
--- a/eclipse.platform.releng.prereqs.sdk/eclipse-sdk-prereqs.target
+++ b/eclipse.platform.releng.prereqs.sdk/eclipse-sdk-prereqs.target
@@ -3,7 +3,7 @@
 
 <target name="eclipse-sdk-prereqs">
 <locations>
-<location includeAllPlatforms="false" includeMode="slicer" type="InstallableUnit">
+<location includeAllPlatforms="true" includeMode="slicer" type="InstallableUnit">
 <unit id="com.ibm.icu" version="50.1.1.v201304230130" />
 <unit id="com.ibm.icu.source" version="50.1.1.v201304230130" />
 <unit id="com.jcraft.jsch" version="0.1.50.v201310081430" />
@@ -84,5 +84,75 @@
 
 <repository location="http://download.eclipse.org/tools/orbit/downloads/drops/M20131015031433/repository/"/>
 </location>
+
+<location includeAllPlatforms="true" includeMode="slicer" type="InstallableUnit">
+<unit id="org.eclipse.emf.common.feature.group" version="2.9.1.v20130827-0309" />
+<unit id="org.eclipse.emf.common.source.feature.group" version="2.9.1.v20130827-0309" />
+<unit id="org.eclipse.emf.ecore.feature.group" version="2.9.1.v20130827-0309" />
+<unit id="org.eclipse.emf.ecore.source.feature.group" version="2.9.1.v20130827-0309" />
+<repository location="http://download.eclipse.org/modeling/emf/emf/updates/2.9.x/base/S201308270309/" />
+</location>
+
+<location includeAllPlatforms="true" includeConfigurePhase="false" includeMode="slicer" includeSource="false" type="InstallableUnit">
+<unit id="org.eclipse.jetty.continuation" version="8.1.10.v20130312"/>
+<unit id="org.eclipse.jetty.continuation.source" version="8.1.10.v20130312"/>
+<unit id="org.eclipse.jetty.http" version="8.1.10.v20130312"/>
+<unit id="org.eclipse.jetty.http.source" version="8.1.10.v20130312"/>
+<unit id="org.eclipse.jetty.io" version="8.1.10.v20130312"/>
+<unit id="org.eclipse.jetty.io.source" version="8.1.10.v20130312"/>
+<unit id="org.eclipse.jetty.security" version="8.1.10.v20130312"/>
+<unit id="org.eclipse.jetty.security.source" version="8.1.10.v20130312"/>
+<unit id="org.eclipse.jetty.server" version="8.1.10.v20130312"/>
+<unit id="org.eclipse.jetty.server.source" version="8.1.10.v20130312"/>
+<unit id="org.eclipse.jetty.servlet" version="8.1.10.v20130312"/>
+<unit id="org.eclipse.jetty.servlet.source" version="8.1.10.v20130312"/>
+<unit id="org.eclipse.jetty.util" version="8.1.10.v20130312"/>
+<unit id="org.eclipse.jetty.util.source" version="8.1.10.v20130312"/>
+<repository location="http://download.eclipse.org/jetty/updates/jetty-bundles-8.x/8.1.10.v20130312/"/>
+
+</location>
+
+<location includeAllPlatforms="true" includeConfigurePhase="false" includeMode="slicer" includeSource="false" type="InstallableUnit">
+
+<unit id="org.eclipse.ecf" version="3.2.0.v20130604-1622" />
+<unit id="org.eclipse.ecf.filetransfer" version="5.0.0.v20130604-1622" />
+<unit id="org.eclipse.ecf.filetransfer.source" version="5.0.0.v20130604-1622" />
+<unit id="org.eclipse.ecf.identity" version="3.2.0.v20130604-1622" />
+<unit id="org.eclipse.ecf.identity.source" version="3.2.0.v20130604-1622" />
+<unit id="org.eclipse.ecf.provider.filetransfer" version="3.2.0.v20130604-1622" />
+<unit id="org.eclipse.ecf.provider.filetransfer.httpclient4" version="1.0.300.v20130604-1622" />
+<unit id="org.eclipse.ecf.provider.filetransfer.httpclient4.source" version="1.0.300.v20130604-1622" />
+<unit id="org.eclipse.ecf.provider.filetransfer.httpclient4.ssl" version="1.0.0.v20130604-1622" />
+<unit id="org.eclipse.ecf.provider.filetransfer.httpclient4.ssl.source" version="1.0.0.v20130604-1622" />
+<unit id="org.eclipse.ecf.provider.filetransfer.source" version="3.2.0.v20130604-1622" />
+<unit id="org.eclipse.ecf.provider.filetransfer.ssl" version="1.0.0.v20130604-1622" />
+<unit id="org.eclipse.ecf.provider.filetransfer.ssl.source" version="1.0.0.v20130604-1622" />
+<unit id="org.eclipse.ecf.source" version="3.2.0.v20130604-1622" />
+<unit id="org.eclipse.ecf.ssl" version="1.1.0.v20130604-1622" />
+<unit id="org.eclipse.ecf.ssl.source" version="1.1.0.v20130604-1622" />
+
+<repository location="http://download.eclipse.org/rt/ecf/int7/site.p2/"/>
+
+</location>
+
+<location includeAllPlatforms="true" includeMode="slicer" type="InstallableUnit">
+<unit id="org.eclipse.license.feature.group" version="1.0.0.v20131003-1638" />
+<repository location="http://download.eclipse.org/cbi/updates/license/1.0.0.v20131003-1638/" />
+</location>
+
+<location includeAllPlatforms="true" includeMode="slicer" type="InstallableUnit">
+<unit id="org.aspectj.runtime" version="1.7.3.20130403143700" />
+<unit id="org.aspectj.weaver" version="1.7.3.20130403143700" />
+<repository location="http://download.eclipse.org/tools/ajdt/aspectj/update/e42/" />
+</location>
+
+<location includeAllPlatforms="true" includeMode="slicer" type="InstallableUnit">
+<unit id="org.eclipse.egit.core" version="3.0.0.201305080800-m7" />
+<unit id="org.eclipse.jgit" version="3.0.0.201305080800-m7" />
+<unit id="javaewah" version="0.5.6.v201210150900" />
+<repository location="http://download.eclipse.org/egit/staging/v3.0.0.201305080800-m7/" />
+</location>
+
+
 </locations>
 </target>
\ No newline at end of file
diff --git a/production/build-functions.shsource b/production/build-functions.shsource
index f542bb6fb..0ace83add 100755
--- a/production/build-functions.shsource
+++ b/production/build-functions.shsource
@@ -291,7 +291,7 @@ fn-local-repo ()
     # we assume REPO_AND_ACCESS is exported from calling scripts
     URL="$1"; shift
     TO_REPLACE='git://git.eclipse.org/gitroot'
-    if [[ -n ${REPO_AND_ACCESS} && ${REPO_AND_ACCESS} != "${TO_REPLACE}" ]]
+    if [[ -n "${REPO_AND_ACCESS}" && "${REPO_AND_ACCESS}" != "${TO_REPLACE}" ]]
     then
         echo $URL | sed "s!${TO_REPLACE}!${REPO_AND_ACCESS}!g"
     else
@@ -465,17 +465,16 @@ fn-basebuilder-dir ()
 }
 
 
-# USAGE: fn-maven-build-aggregator BUILD_ID REPO_DIR LOCAL_REPO DEBUG QUIET SIGNING UPDATE_BRANDING MAVEN_BREE
+# USAGE: fn-maven-build-aggregator BUILD_ID REPO_DIR LOCAL_REPO DEBUG QUIET SIGNING MAVEN_BREE
 #   BUILD_ID: I20121116-0700
 #   REPO_DIR: /shared/eclipse/builds/R4_2_maintenance/gitCache/eclipse.platform.releng.aggregator
 #   LOCAL_REPO: /shared/eclipse/builds/R4_2_maintenance/localMavenRepo
 #   VERBOSE: true
 #   SIGNING: true
-#   UPDATE_BRANDING: true
 # TODO: depends on BUILD_TYPE being exported/global
 fn-maven-build-aggregator () 
 {
-    checkNArgs $# 8
+    checkNArgs $# 7
     if [[ $? != 0 ]]; then return 1; fi
     BUILD_ID="$1"; shift
     REPO_DIR="$1"; shift
@@ -483,7 +482,6 @@ fn-maven-build-aggregator ()
     DEBUG=$1; shift
     QUIET=$1; shift
     SIGNING=$1; shift
-    UPDATE_BRANDING=$1; shift
     MAVEN_BREE=$1; shift
 
     MARGS="-DbuildId=$BUILD_ID"
@@ -496,9 +494,7 @@ fn-maven-build-aggregator ()
     if $SIGNING; then
         MARGS="$MARGS -Peclipse-sign"
     fi
-    if $UPDATE_BRANDING; then
-        MARGS="$MARGS -Pupdate-branding-plugins"
-    fi
+
     MARGS="$MARGS ${MAVEN_BREE}"
 
     # Here we count on $BUILD_TYPE being exported. TODO: make parameter later? 
@@ -517,7 +513,6 @@ fn-maven-build-aggregator ()
     echo "DEBUG: DEBUG: $DEBUG"
     echo "DEBUG: QUIET: $QUIET"
     echo "DEBUG: SIGNING: $SIGNING"
-    echo "DEBUG: UPDATE_BRANDING: $UPDATE_BRANDING"
     echo "DEBUG: MAVEN_BREE: $MAVEN_BREE"
     echo "DEBUG: MARGS: $MARGS"
     echo "DEBUG: FORCEQUALIFIERARG: $FORCEQUALIFIERARG"
@@ -604,9 +599,9 @@ fn-tag-build-inputs ()
     REPO_DIR="$1"; shift
     REPOSITORIES_TXT="$1"; shift
     pushd "$REPO_DIR"
-    git submodule foreach "if grep \"^\${name}:\" $REPOSITORIES_TXT >/dev/null; then git tag $BUILD_ID; $GIT_PUSH origin $BUILD_ID; else echo Skipping \$name; fi"
+    git submodule foreach "if grep \"^\${name}:\" $REPOSITORIES_TXT >/dev/null; then git tag $BUILD_ID; $GIT_PUSH --verbose origin $BUILD_ID; else echo Skipping \$name; fi"
     git tag $BUILD_ID
-    $GIT_PUSH origin $BUILD_ID
+    $GIT_PUSH --verbose origin $BUILD_ID
     popd
 }
 
diff --git a/production/build_eclipse_org.shsource b/production/build_eclipse_org.shsource
index 59ccc6f6c..f7234afc8 100644
--- a/production/build_eclipse_org.shsource
+++ b/production/build_eclipse_org.shsource
@@ -37,8 +37,17 @@ export MAVEN_PATH=${MAVEN_PATH:-/shared/common/apache-maven-3.1.0/bin}
 
 export PATH=$JAVA_HOME/bin:$MAVEN_PATH:$ANT_HOME/bin:$PATH
 
-# REPO_AND_ACCESS normally defined by now, by bootstrap, but if not, we assume git access to eclipse.org
-export REPO_AND_ACCESS=${REPO_AND_ACCESS:-git://git.eclipse.org/gitroot}
+# if not defined "externally", we use default for eclipse.org
+if [[ -z "${REPO_AND_ACCESS}" ]]
+then
+    # unless we are on 'build' machine
+    if [[ "build" == "$(hostname)" ]]
+    then
+        export REPO_AND_ACCESS=file:///gitroot
+    else
+        export REPO_AND_ACCESS=git://git.eclipse.org/gitroot
+    fi
+fi
 
 AGGREGATOR_REPO=${REPO_AND_ACCESS}/platform/eclipse.platform.releng.aggregator.git
 
@@ -52,7 +61,6 @@ COMMITTER_ID=e4Build
 # MVN_DEBUG=true means verbose; gives comparator info
 export MVN_DEBUG=${MVN_DEBUG:-true}
 export MVN_QUIET=${MVN_QUIET:-false}
-export UPDATE_BRANDING=true
 
 # Need bree-libs for production builds
 MAVEN_BREE=-Pbree-libs
diff --git a/production/getEBuilderForDropDir.sh b/production/getEBuilderForDropDir.sh
index 11f77b8b0..2e216dbc1 100755
--- a/production/getEBuilderForDropDir.sh
+++ b/production/getEBuilderForDropDir.sh
@@ -33,6 +33,10 @@ if [[ ! -d ${BUILD_DIR}/${EBUILDER} ]]
 then
     # Not sure 'REPO_AND_ACCESS' is defined in all possible scenerios, so we'll provide a default. 
     # It is in main scenerios, but not sure about things like "re-running unit tests at a later time".
+    # If directory doesn't exist yet, create it first, so we can assign proper access 
+    # permissions for later "clean up" routines.
+    mkdir -p "${BUILD_DIR}/${EBUILDER}"
+    chmod -c g+ws "${BUILD_DIR}/${EBUILDER}"
     git clone ${REPO_AND_ACCESS:-git://git.eclipse.org/gitroot}/platform/${EBUILDER} ${BUILD_DIR}/${EBUILDER}
     RC=$?
     if [[ $RC != 0 ]] 
diff --git a/production/master-build.sh b/production/master-build.sh
index 619139bb6..c53e77338 100644
--- a/production/master-build.sh
+++ b/production/master-build.sh
@@ -71,6 +71,12 @@ if [[ -z "${buildDirectory}" ]]
 then
     echo "PROGRAM ERROR: buildDirectory returned from fn-build-dir was empty"
     exit 1
+else 
+    # this should be when we first create buildDirectory
+    echo "Making buildDirectory: ${buildDirectory}"
+    mkdir -p "${buildDirectory}"
+    # it appears GID bit is not always set correctly, so we'll do so explicitly
+    chmod -c g+s "${buildDirectory}"
 fi
 export logsDirectory="${buildDirectory}/buildlogs"
 mkdir -p "${logsDirectory}"
@@ -100,12 +106,14 @@ gitCache=$( fn-git-cache "$BUILD_ROOT" "$BRANCH" )
 aggDir=$( fn-git-dir "$gitCache" "$AGGREGATOR_REPO" )
 export LOCAL_REPO="${BUILD_ROOT}"/localMavenRepo
 
-# for now, remove any existing LOCAL_REPO, and re-fetch. 
-# At some point we may reconsider this only remove once a week, 
-# or something. 
+# In production builds, we normally specify CLEAN_LOCAL, 
+# and remove any existing LOCAL_REPO, and re-fetch. 
+# But CLEAN_LOCAL can be overridden for remote builds, quick test builds, 
+# etc. 
+export CLEAN_LOCAL=${CLEAN_LOCAL:-true}
 # We "remove" by moving to backup, in case there's ever any reason to 
 # compare "what's changed". 
-if [[ -d ${LOCAL_REPO} ]]
+if [[ -d ${LOCAL_REPO} && "${CLEAN_LOCAL}" == "true" ]]
 then
     # remove existing backup, if it exists
     rm -fr ${LOCAL_REPO}.bak 2>/dev/null
@@ -164,7 +172,6 @@ fn-write-property COMMITTER_ID
 fn-write-property MVN_DEBUG
 fn-write-property MVN_QUIET
 fn-write-property SIGNING
-fn-write-property UPDATE_BRANDING
 fn-write-property REPO_AND_ACCESS
 fn-write-property MAVEN_BREE
 fn-write-property GIT_PUSH
diff --git a/production/publish-equinox.sh b/production/publish-equinox.sh
index be14c6dd8..2f40356c5 100644
--- a/production/publish-equinox.sh
+++ b/production/publish-equinox.sh
@@ -112,7 +112,7 @@ fn-publish-equinox ()
     REPO_DIR="$1"; shift
     BUILD_DIR="$1"; shift
     BASEBUILDER_LAUNCHER="$1"; shift
-    BUILD_MACHINE_ROOT=/shared/eclipse/builds
+    BUILD_MACHINE_ROOT=${BUILD_HOME}
     BUILD_MACHINE_DROP_DIR=$(fn-eq-build-dir "$BUILD_MACHINE_ROOT" "$BUILD_ID" "$BUILD_STREAM")
     BUILD_MACHINE_DROP_DIR_PARENT=$(dirname $BUILD_MACHINE_DROP_DIR)
     EBuilderDir="$BUILD_DIR"/eclipse.platform.releng.aggregator/eclipse.platform.releng.tychoeclipsebuilder
diff --git a/production/run-maven-build.sh b/production/run-maven-build.sh
index fabc119e5..9a331bbc1 100644
--- a/production/run-maven-build.sh
+++ b/production/run-maven-build.sh
@@ -29,7 +29,7 @@ if [ -z "$BUILD_ID" ]; then
     BUILD_ID=$(fn-build-id "$BUILD_TYPE" )
 fi
 
-fn-maven-build-aggregator "$BUILD_ID" "$aggDir" "$LOCAL_REPO" $MVN_DEBUG $MVN_QUIET $SIGNING $UPDATE_BRANDING $MAVEN_BREE
+fn-maven-build-aggregator "$BUILD_ID" "$aggDir" "$LOCAL_REPO" $MVN_DEBUG $MVN_QUIET $SIGNING $MAVEN_BREE
 exitCode=$?
 
 # first make sure exit code is well formed