blob: ab1b4942aacb3ef4240d09ccb8864b95fed8bbdf [file] [log] [blame]
Stephan Herrmannee73d532016-12-20 23:37:28 +01001#!/bin/bash
2#*******************************************************************************
Stephan Herrmann95fbf622017-03-05 13:43:09 +01003# Copyright (c) 2016, 2017 GK Software AG and others.
Stephan Herrmannee73d532016-12-20 23:37:28 +01004# All rights reserved. This program and the accompanying materials
5# are made available under the terms of the Eclipse Public License v1.0
6# which accompanies this distribution, and is available at
7# http://www.eclipse.org/legal/epl-v10.html
8#
9# Contributors:
10# Stephan Herrmann - initial API and implementation
11#********************************************************************************
12
13#================================================================================
14# Parameters we might want to externalize into a properties file
15#================================================================================
16
Stephan Herrmann4e843b02017-01-26 21:18:42 +010017source `dirname ${0}`/properties.sh
Stephan Herrmannee73d532016-12-20 23:37:28 +010018
Stephan Herrmannee73d532016-12-20 23:37:28 +010019
20#================================================================================
21# Util functions
22#================================================================================
23function require_executable() {
24 if [ -x ${1} ]
25 then
26 echo "Successfully installed: ${1}"
27 else
28 echo "not executable: ${1}"
29 /bin/ls -l ${1}
30 exit 1
31 fi
32}
33
Stephan Herrmann0d566b52017-10-17 17:58:14 +020034function create_baseline() {
35 cd ${Repo}
Stephan Herrmann31cf37f2017-10-17 21:56:31 +020036 for line in `find org/eclipse -name \*[0-9].jar`
Stephan Herrmann0d566b52017-10-17 17:58:14 +020037 do
38 file=`basename $line`
Stephan Herrmann5e3f5622017-10-17 22:20:42 +020039 name=`echo $file | sed -e 's/\(.*\)-.*/\1/' | tr '.' '_'`
40 version=`echo $file | sed -e 's/.*-\(.*\)\.jar/\1/'`
Stephan Herrmannc4ae0f32017-10-17 20:54:09 +020041 previous=`eval echo \\${VERSION_$name}`
Stephan Herrmann5e3f5622017-10-17 22:20:42 +020042 if [ "$previous" != "" ]
Stephan Herrmannc4ae0f32017-10-17 20:54:09 +020043 then
44 version=${previous},${version}
45 fi
46 eval VERSION_$name=$version
Stephan Herrmann0d566b52017-10-17 17:58:14 +020047 echo VERSION_$name=$version
48 done > ${WORKSPACE}/baseline.txt
49 cd -
50}
51
Stephan Herrmann9a981012017-01-01 11:05:54 +010052#================================================================================
53# (1) Install and run the CBI aggregator
54#================================================================================
55echo "==== CBI aggregator ===="
56
Stephan Herrmannb3148c62016-12-21 00:42:13 +010057# -------- fetch .aggr file **TEMP** will eventually move to the releng git: ------------
58git archive --remote=file://localhost/gitroot/objectteams/org.eclipse.objectteams.git \
59 master releng/EnrichPoms/SDK4Mvn.aggr \
60 | tar xv
61/bin/mv releng/EnrichPoms/SDK4Mvn.aggr ${WORKSPACE}/
62/bin/rmdir -p releng/EnrichPoms
63
Stephan Herrmannee73d532016-12-20 23:37:28 +010064if [ ! -d ${LOCAL_TOOLS} ]
65then
66 /bin/mkdir ${LOCAL_TOOLS}
67fi
68
69if [ ! -x ${ECLIPSE} ]
70then
71 cd ${LOCAL_TOOLS}
Stephan Herrmann9a981012017-01-01 11:05:54 +010072 echo "Extracting Eclipse from ${FILE_ECLIPSE} ..."
Stephan Herrmannee73d532016-12-20 23:37:28 +010073 tar xf ${FILE_ECLIPSE}
74 cd ${WORKSPACE}
75fi
76require_executable ${ECLIPSE}
77
78if [ ! -x ${AGGREGATOR} ]
79then
Stephan Herrmann9a981012017-01-01 11:05:54 +010080 echo "Installing the CBI aggregator into ${LOCAL_TOOLS}/${DIR_AGGREGATOR} ..."
Stephan Herrmannee73d532016-12-20 23:37:28 +010081 ${ECLIPSE} -application ${APP_NAME_P2DIRECTOR} \
82 -r ${URL_AGG_UPDATES} \
83 -d ${LOCAL_TOOLS}/${DIR_AGGREGATOR} -p CBIProfile \
84 -installIU ${IU_AGG_PRODUCT}
85fi
86require_executable ${AGGREGATOR}
87
88RepoRaw=${WORKSPACE}/reporaw-${BUILD_NUMBER}
89Repo=${WORKSPACE}/repo-${BUILD_NUMBER}
90/bin/mkdir ${RepoRaw}
91
Stephan Herrmann9a981012017-01-01 11:05:54 +010092echo "Running the aggregator with build model ${FILE_SDK_AGGR} ..."
Stephan Herrmannee73d532016-12-20 23:37:28 +010093${AGGREGATOR} aggregate --buildModel ${FILE_SDK_AGGR} --action CLEAN_BUILD --buildRoot ${RepoRaw}
94if [ "$?" != "0" ]
95then
96 echo "FAILURE $?"
97 exit 1
98fi
99/bin/mv ${RepoRaw}/final ${Repo}
100/bin/rm -rf ${RepoRaw}
101
102echo "========== Repo created: =========="
103/usr/bin/du -sc ${Repo}/*
104/usr/bin/du -sc ${Repo}/org/*
105/usr/bin/du -sc ${Repo}/org/eclipse/*
106echo "==================================="
107
108
109#================================================================================
110# (2) Remove irrelevant stuff
111#================================================================================
112# Removes from the build output of cbiAggregator everything that is not relevant for maven.
113# All removed directories / files will be logged to .logs/removed.txt
114
115echo "==== Remove irrelevant stuff ===="
116
117cd ${Repo}
118
119if [ ! -d .logs ]
120then
121 /bin/mkdir .logs
122elif [ -f .logs/removed.txt ]
123then
124 /bin/rm .logs/removed.txt
125fi
126
127#==== remove the p2 repository (not logged): ====
128
129/bin/rm -r p2.index p2.packed content.jar artifacts.jar
130
Stephan Herrmannee73d532016-12-20 23:37:28 +0100131#==== remove features: ====
132
133echo "== Features: ==" | tee >> .logs/removed.txt
134
135/usr/bin/find * -type d -name \*feature.group -print -exec /bin/rm -rf {} \; -prune >> .logs/removed.txt
136/usr/bin/find * -type d -name \*feature.jar -print -exec /bin/rm -rf {} \; -prune >> .logs/removed.txt
137
138#==== remove eclipse test plug-ins: ====
139
140echo "== Test plugins: ==" | tee >> .logs/removed.txt
141
142ls -d org/eclipse/*/*test* >> .logs/removed.txt
143/bin/rm -r org/eclipse/*/*test*
144
145#==== remove other non-artifacts: ====
146
147echo "== Other non-artifacts: ==" | tee >> .logs/removed.txt
148
149/usr/bin/find tooling -type d >> .logs/removed.txt
150/bin/rm -r tooling*
151
152# ... folders that contain only 1.2.3/foo-1.2.3.pom but no corresponding 1.2.3/foo-1.2.3.jar:
153function hasPomButNoJar() {
154 cd ${1}
155 # expect only one sub-directory, starting with a digit, plus maven-metadata.xml*:
156 other=`ls -d [!0-9]* 2> /dev/null`
Stephan Herrmann0b6f3932017-01-01 21:17:57 +0100157 if echo "${other}" | tr "\n" " " | egrep "^maven-metadata.xml maven-metadata.xml.md5 maven-metadata.xml.sha1 \$"
Stephan Herrmannee73d532016-12-20 23:37:28 +0100158 then
Stephan Herrmann0a358922017-01-01 11:19:36 +0100159 : # clean -> proceed below
Stephan Herrmann9a981012017-01-01 11:05:54 +0100160 else
161 exit 1 # unexpected content found, don't remove
Stephan Herrmannee73d532016-12-20 23:37:28 +0100162 fi
163 # scan all *.pom inside the version sub-directory
164 r=1
165 for pom in `ls [0-9]*/*.pom 2> /dev/null`
166 do
167 jar=`echo ${pom} | sed -e "s|\(.*\)\.pom|\1.jar|"`
168 if [ -f ${jar} ]
169 then
170 # jar found, so keep it
171 exit 1
172 fi
173 # pom without jar found, let's answer true below
174 r=0
175 done
176 exit $r
177}
178export -f hasPomButNoJar
179
180/usr/bin/find org/eclipse/{jdt,pde,platform} -type d \
181 -exec /bin/bash -c 'hasPomButNoJar "$@"' bash {} \; \
182 -print -exec /bin/rm -rf {} \; -prune >> .logs/removed.txt
183# second "bash" is used as $0 in the function
184
185cd ${WORKSPACE}
186
187echo "========== Repo reduced: =========="
188/usr/bin/du -sc ${Repo}/*
189/usr/bin/du -sc ${Repo}/org/*
190/usr/bin/du -sc ${Repo}/org/eclipse/*
191echo "==================================="
192
193#================================================================================
194# (2) Garbage Collector
195#================================================================================
196# Removes from the build output of cbiAggregator everything that is not referenced
197# from any pom below org/eclipse/{platform,jdt,pde}
198#
199# Log output:
200# .logs/removedGarbage.txt all directories during garbage collection
201# .logs/gc.log incoming dependencies of retained artifacts
202# .logs/empty-dirs.txt removed empty directories
203
204echo "==== Garbage Collector ===="
205
206cd ${Repo}
207
208#==== function gc_bundle(): ====
209# Test if pom ${1} is referenced in any other pom.
210# If not, append the containing directory to the file "toremove.txt"
211function gc_bundle {
212 AID=`echo ${1} | sed -e "s|.*/\(.*\)[_-].*|\1|"`
213 DIR=`echo ${1} | sed -e "s|\(.*\)/[0-9].*|\1|"`
214 POM=`basename ${1}`
215
216 ANSWER=`find org/eclipse/{platform,jdt,pde} -name \*.pom \! -name ${POM} \
217 -exec /bin/grep -q "<artifactId>${AID}</artifactId>" {} \; -print -quit`
218
219 if [ "$ANSWER" == "" ]
220 then
221 echo "Will remove $DIR"
222 echo $DIR >> toremove.txt
223 else
224 echo "$1 is used by $ANSWER"
225 fi
226}
227export -f gc_bundle
228
229#==== run the garbage collector: ====
230# iterate (max 5 times) in case artifacts were used only from garbage:
231for iteration in 1 2 3 4 5
232do
233 echo "== GC iteration ${iteration} =="
234
235 # look for garbage only outside platform, jdt or pde folders:
236 find -name platform -prune -o -name jdt -prune -o -name pde -prune -o \
237 -name \*.pom -exec /bin/bash -c 'gc_bundle "$@"' bash {} \; \
238 > gc-${iteration}.log
239 # second "bash" is used as $0 in the function
240
241 if [ ! -f toremove.txt ]
242 then
243 # no more garbage found
244 break
245 fi
246 cat toremove.txt >> .logs/removedGarbage.txt
247 for d in `cat toremove.txt`; do /bin/rm -r $d; done
248 /bin/rm toremove.txt
249done
250
251# merge gc logs:
252cat gc-*.log | sort --unique > .logs/gc.log
253/bin/rm gc-*.log
254
255#==== remove all directories that have become empty: ====
Stephan Herrmann3b557672017-01-26 22:26:12 +0100256for iteration in 1 2 3 4 5 6 ; do find -type d -empty -print \
Stephan Herrmannee73d532016-12-20 23:37:28 +0100257 -exec /bin/rmdir {} \; -prune; done \
258 >> .logs/empty-dirs.txt
259
260echo "========== Repo reduced: =========="
261/usr/bin/du -sc ${Repo}/*
262/usr/bin/du -sc ${Repo}/org/*
263/usr/bin/du -sc ${Repo}/org/eclipse/*
264echo "==================================="
265
266cd ${WORKSPACE}
267
268#================================================================================
269# (3) Enrich POMs
270#================================================================================
271# Add some required information to the generated poms:
272# - dynamic content (retrieved mostly from MANIFEST.MF):
273# - name
274# - url
Stephan Herrmann9a981012017-01-01 11:05:54 +0100275# - scm connection, tag and url
276# - semi dynamic
277# - developers (based on static map git-repo-base -> project leads)
Stephan Herrmannee73d532016-12-20 23:37:28 +0100278# - static content
279# - license
280# - organization
281# - issue management
282
283
284echo "==== Enrich POMs ===="
285
286cd ${Repo}
287
288echo "platform"
289java -jar ${ENRICH_POMS_JAR} `pwd`/org/eclipse/platform &> .logs/enrich-platform.txt
290echo "jdt"
291java -jar ${ENRICH_POMS_JAR} `pwd`/org/eclipse/jdt &> .logs/enrich-jdt.txt
292echo "pde"
293java -jar ${ENRICH_POMS_JAR} `pwd`/org/eclipse/pde &> .logs/enrich-pde.txt
294
Stephan Herrmannee73d532016-12-20 23:37:28 +0100295
Stephan Herrmann9a981012017-01-01 11:05:54 +0100296echo "==== Add Javadoc stubs ===="
Stephan Herrmann42f24c92016-12-31 16:53:10 +0100297
Stephan Herrmann07ae09f2017-01-01 18:58:28 +0100298cd ${Repo}
299
Stephan Herrmann42f24c92016-12-31 16:53:10 +0100300# (groupSimpleName, javadocArtifactGA)
301function createJavadocs() {
302 group=${1}
303 jar="${1}-javadoc.jar"
304 artifact=${2}
305 if [ -r ${jar} ]
306 then
307 /bin/rm ${jar}
308 fi
Stephan Herrmann9fc05c92016-12-31 17:35:26 +0100309 echo -e "Corresponding javadoc can be found in artifact ${artifact}\n" > README.txt
Stephan Herrmann42f24c92016-12-31 16:53:10 +0100310 jar cf ${jar} README.txt
311 for pom in org/eclipse/${group}/*/*/*.pom
312 do
313 javadoc=`echo ${pom} | sed -e "s|\(.*\)\.pom|\1-javadoc.jar|"`
314 /bin/cp ${jar} ${javadoc}
315 done
316}
317
318createJavadocs platform org.eclipse.platform:org.eclipse.platform.doc.isv
319createJavadocs jdt org.eclipse.jdt:org.eclipse.jdt.doc.isv
320createJavadocs pde org.eclipse.pde:org.eclipse.pde.doc.user
Stephan Herrmannee73d532016-12-20 23:37:28 +0100321
Stephan Herrmann07ae09f2017-01-01 18:58:28 +0100322echo "==== Create missing sources artifacts ===="
323
324cd ${Repo}
325
326function buildSourceJar() {
327 if [ -d assemble ]
328 then
329 /bin/rm -r assemble
330 fi
331 /bin/mkdir assemble
Stephan Herrmannc6c07eb2017-01-01 20:50:11 +0100332 giturl=file://localhost/gitroot/${1}
Stephan Herrmann07ae09f2017-01-01 18:58:28 +0100333 gitpath=${2}
334 gittag=${3}
335 group=${4}
336 artifact=${5}
337 version=${6}
338 git archive --remote=${giturl} \
339 ${gittag} ${gitpath} \
340 | tar xv
341 /bin/mv ${gitpath}/src/* assemble/
342 /bin/mv ${gitpath}/META-INF assemble/
343 if [ -d ${gitpath}/OSGI-INF ]
344 then
345 /bin/mv ${gitpath}/OSGI-INF assemble/
346 fi
347 /bin/mv ${gitpath}/about.html assemble/
Stephan Herrmannc6c07eb2017-01-01 20:50:11 +0100348 if [ $# > 6 ]
349 then
350 shift 6
351 for src in "$@"
352 do
353 /bin/mv "${gitpath}/${src}" assemble/
354 done
355 fi
Stephan Herrmann07ae09f2017-01-01 18:58:28 +0100356 /bin/rm -rf ${gitpath}
357 cd assemble
358 jar cf ../${group}/${artifact}/${version}/${artifact}-${version}-sources.jar *
359 cd -
360}
361
Stephan Herrmann4e843b02017-01-26 21:18:42 +0100362while read line
363do
364 buildSourceJar $line
Stephan Herrmann3b557672017-01-26 22:26:12 +0100365done < ${WORKSPACE}/sourceBundles.txt
Stephan Herrmann07ae09f2017-01-01 18:58:28 +0100366
Stephan Herrmann95fbf622017-03-05 13:43:09 +0100367# copy ecj-src from build output (NB: we are mapping from SDK version (4.x) back to ECJ version (3.x)):
Stephan Herrmann9fb254e2017-03-05 13:53:22 +0100368/bin/cp ${DROPS4}/${SDK_BUILD_DIR}/ecjsrc-${SDK_VERSION}.jar org/eclipse/jdt/ecj/${ECJ_VERSION}/ecj-${ECJ_VERSION}-sources.jar
Stephan Herrmann95fbf622017-03-05 13:43:09 +0100369
Stephan Herrmann4e843b02017-01-26 21:18:42 +0100370# special hack for missing source bundle with several source folders with blanks:
Stephan Herrmannc6c07eb2017-01-01 20:50:11 +0100371buildSourceJar platform/eclipse.platform.swt.git \
372 bundles/org.eclipse.swt.tools \
Stephan Herrmann99ad6fb2017-07-26 21:50:23 +0200373 R4_7 \
374 org/eclipse/platform org.eclipse.swt.tools 3.106.0 \
Stephan Herrmannc6c07eb2017-01-01 20:50:11 +0100375 "JNI Generation" \
376 "Mac Generation" \
377 "Icon Exe" \
378 "NativeStats" \
379 "Mozilla Generation" \
380 "JavadocBasher"
381
Stephan Herrmannee73d532016-12-20 23:37:28 +0100382echo "========== Repo completed ========="
383
Stephan Herrmann0d566b52017-10-17 17:58:14 +0200384create_baseline
385
386echo "========== Next baseline created ========="
387
Stephan Herrmannee73d532016-12-20 23:37:28 +0100388cd ${WORKSPACE}