blob: ffcf6534f23b23523ce47b04615412594e8e3a2b [file] [log] [blame]
Stephan Herrmannee73d532016-12-20 23:37:28 +01001#!/bin/bash
2#*******************************************************************************
3# Copyright (c) 2016 GK Software AG and others.
4# All rights reserved. This program and the accompanying materials
5# are made available under the terms of the Eclipse Public License v1.0
6# which accompanies this distribution, and is available at
7# http://www.eclipse.org/legal/epl-v10.html
8#
9# Contributors:
10# Stephan Herrmann - initial API and implementation
11#********************************************************************************
12
13#================================================================================
14# Parameters we might want to externalize into a properties file
15#================================================================================
16
Stephan Herrmann4e843b02017-01-26 21:18:42 +010017source `dirname ${0}`/properties.sh
Stephan Herrmannee73d532016-12-20 23:37:28 +010018
Stephan Herrmannee73d532016-12-20 23:37:28 +010019
20#================================================================================
21# Util functions
22#================================================================================
23function require_executable() {
24 if [ -x ${1} ]
25 then
26 echo "Successfully installed: ${1}"
27 else
28 echo "not executable: ${1}"
29 /bin/ls -l ${1}
30 exit 1
31 fi
32}
33
Stephan Herrmann9a981012017-01-01 11:05:54 +010034#================================================================================
35# (1) Install and run the CBI aggregator
36#================================================================================
37echo "==== CBI aggregator ===="
38
Stephan Herrmannb3148c62016-12-21 00:42:13 +010039# -------- fetch .aggr file **TEMP** will eventually move to the releng git: ------------
40git archive --remote=file://localhost/gitroot/objectteams/org.eclipse.objectteams.git \
41 master releng/EnrichPoms/SDK4Mvn.aggr \
42 | tar xv
43/bin/mv releng/EnrichPoms/SDK4Mvn.aggr ${WORKSPACE}/
44/bin/rmdir -p releng/EnrichPoms
45
Stephan Herrmannee73d532016-12-20 23:37:28 +010046if [ ! -d ${LOCAL_TOOLS} ]
47then
48 /bin/mkdir ${LOCAL_TOOLS}
49fi
50
51if [ ! -x ${ECLIPSE} ]
52then
53 cd ${LOCAL_TOOLS}
Stephan Herrmann9a981012017-01-01 11:05:54 +010054 echo "Extracting Eclipse from ${FILE_ECLIPSE} ..."
Stephan Herrmannee73d532016-12-20 23:37:28 +010055 tar xf ${FILE_ECLIPSE}
56 cd ${WORKSPACE}
57fi
58require_executable ${ECLIPSE}
59
60if [ ! -x ${AGGREGATOR} ]
61then
Stephan Herrmann9a981012017-01-01 11:05:54 +010062 echo "Installing the CBI aggregator into ${LOCAL_TOOLS}/${DIR_AGGREGATOR} ..."
Stephan Herrmannee73d532016-12-20 23:37:28 +010063 ${ECLIPSE} -application ${APP_NAME_P2DIRECTOR} \
64 -r ${URL_AGG_UPDATES} \
65 -d ${LOCAL_TOOLS}/${DIR_AGGREGATOR} -p CBIProfile \
66 -installIU ${IU_AGG_PRODUCT}
67fi
68require_executable ${AGGREGATOR}
69
70RepoRaw=${WORKSPACE}/reporaw-${BUILD_NUMBER}
71Repo=${WORKSPACE}/repo-${BUILD_NUMBER}
72/bin/mkdir ${RepoRaw}
73
Stephan Herrmann9a981012017-01-01 11:05:54 +010074echo "Running the aggregator with build model ${FILE_SDK_AGGR} ..."
Stephan Herrmannee73d532016-12-20 23:37:28 +010075${AGGREGATOR} aggregate --buildModel ${FILE_SDK_AGGR} --action CLEAN_BUILD --buildRoot ${RepoRaw}
76if [ "$?" != "0" ]
77then
78 echo "FAILURE $?"
79 exit 1
80fi
81/bin/mv ${RepoRaw}/final ${Repo}
82/bin/rm -rf ${RepoRaw}
83
84echo "========== Repo created: =========="
85/usr/bin/du -sc ${Repo}/*
86/usr/bin/du -sc ${Repo}/org/*
87/usr/bin/du -sc ${Repo}/org/eclipse/*
88echo "==================================="
89
90
91#================================================================================
92# (2) Remove irrelevant stuff
93#================================================================================
94# Removes from the build output of cbiAggregator everything that is not relevant for maven.
95# All removed directories / files will be logged to .logs/removed.txt
96
97echo "==== Remove irrelevant stuff ===="
98
99cd ${Repo}
100
101if [ ! -d .logs ]
102then
103 /bin/mkdir .logs
104elif [ -f .logs/removed.txt ]
105then
106 /bin/rm .logs/removed.txt
107fi
108
109#==== remove the p2 repository (not logged): ====
110
111/bin/rm -r p2.index p2.packed content.jar artifacts.jar
112
113#==== remove -sources artifacts, misplaced due to quirk from https://bugs.eclipse.org/508910: ====
114
115echo "== Misplaced -sources artifacts: ==" | tee >> .logs/removed.txt
116
117# works only outside org/eclipse/{platform,jdt,pde}:
118
119/usr/bin/find -type d -name platform -prune -o -name jdt -prune -o -name pde -prune -o \
120 -name \*-sources.jar\* -print -exec /bin/rm {} \; >> .logs/removed.txt
121
122#==== remove features: ====
123
124echo "== Features: ==" | tee >> .logs/removed.txt
125
126/usr/bin/find * -type d -name \*feature.group -print -exec /bin/rm -rf {} \; -prune >> .logs/removed.txt
127/usr/bin/find * -type d -name \*feature.jar -print -exec /bin/rm -rf {} \; -prune >> .logs/removed.txt
128
129#==== remove eclipse test plug-ins: ====
130
131echo "== Test plugins: ==" | tee >> .logs/removed.txt
132
133ls -d org/eclipse/*/*test* >> .logs/removed.txt
134/bin/rm -r org/eclipse/*/*test*
135
136#==== remove other non-artifacts: ====
137
138echo "== Other non-artifacts: ==" | tee >> .logs/removed.txt
139
140/usr/bin/find tooling -type d >> .logs/removed.txt
141/bin/rm -r tooling*
142
143# ... folders that contain only 1.2.3/foo-1.2.3.pom but no corresponding 1.2.3/foo-1.2.3.jar:
144function hasPomButNoJar() {
145 cd ${1}
146 # expect only one sub-directory, starting with a digit, plus maven-metadata.xml*:
147 other=`ls -d [!0-9]* 2> /dev/null`
Stephan Herrmann0b6f3932017-01-01 21:17:57 +0100148 if echo "${other}" | tr "\n" " " | egrep "^maven-metadata.xml maven-metadata.xml.md5 maven-metadata.xml.sha1 \$"
Stephan Herrmannee73d532016-12-20 23:37:28 +0100149 then
Stephan Herrmann0a358922017-01-01 11:19:36 +0100150 : # clean -> proceed below
Stephan Herrmann9a981012017-01-01 11:05:54 +0100151 else
152 exit 1 # unexpected content found, don't remove
Stephan Herrmannee73d532016-12-20 23:37:28 +0100153 fi
154 # scan all *.pom inside the version sub-directory
155 r=1
156 for pom in `ls [0-9]*/*.pom 2> /dev/null`
157 do
158 jar=`echo ${pom} | sed -e "s|\(.*\)\.pom|\1.jar|"`
159 if [ -f ${jar} ]
160 then
161 # jar found, so keep it
162 exit 1
163 fi
164 # pom without jar found, let's answer true below
165 r=0
166 done
167 exit $r
168}
169export -f hasPomButNoJar
170
171/usr/bin/find org/eclipse/{jdt,pde,platform} -type d \
172 -exec /bin/bash -c 'hasPomButNoJar "$@"' bash {} \; \
173 -print -exec /bin/rm -rf {} \; -prune >> .logs/removed.txt
174# second "bash" is used as $0 in the function
175
176cd ${WORKSPACE}
177
178echo "========== Repo reduced: =========="
179/usr/bin/du -sc ${Repo}/*
180/usr/bin/du -sc ${Repo}/org/*
181/usr/bin/du -sc ${Repo}/org/eclipse/*
182echo "==================================="
183
184#================================================================================
185# (2) Garbage Collector
186#================================================================================
187# Removes from the build output of cbiAggregator everything that is not referenced
188# from any pom below org/eclipse/{platform,jdt,pde}
189#
190# Log output:
191# .logs/removedGarbage.txt all directories during garbage collection
192# .logs/gc.log incoming dependencies of retained artifacts
193# .logs/empty-dirs.txt removed empty directories
194
195echo "==== Garbage Collector ===="
196
197cd ${Repo}
198
199#==== function gc_bundle(): ====
200# Test if pom ${1} is referenced in any other pom.
201# If not, append the containing directory to the file "toremove.txt"
202function gc_bundle {
203 AID=`echo ${1} | sed -e "s|.*/\(.*\)[_-].*|\1|"`
204 DIR=`echo ${1} | sed -e "s|\(.*\)/[0-9].*|\1|"`
205 POM=`basename ${1}`
206
207 ANSWER=`find org/eclipse/{platform,jdt,pde} -name \*.pom \! -name ${POM} \
208 -exec /bin/grep -q "<artifactId>${AID}</artifactId>" {} \; -print -quit`
209
210 if [ "$ANSWER" == "" ]
211 then
212 echo "Will remove $DIR"
213 echo $DIR >> toremove.txt
214 else
215 echo "$1 is used by $ANSWER"
216 fi
217}
218export -f gc_bundle
219
220#==== run the garbage collector: ====
221# iterate (max 5 times) in case artifacts were used only from garbage:
222for iteration in 1 2 3 4 5
223do
224 echo "== GC iteration ${iteration} =="
225
226 # look for garbage only outside platform, jdt or pde folders:
227 find -name platform -prune -o -name jdt -prune -o -name pde -prune -o \
228 -name \*.pom -exec /bin/bash -c 'gc_bundle "$@"' bash {} \; \
229 > gc-${iteration}.log
230 # second "bash" is used as $0 in the function
231
232 if [ ! -f toremove.txt ]
233 then
234 # no more garbage found
235 break
236 fi
237 cat toremove.txt >> .logs/removedGarbage.txt
238 for d in `cat toremove.txt`; do /bin/rm -r $d; done
239 /bin/rm toremove.txt
240done
241
242# merge gc logs:
243cat gc-*.log | sort --unique > .logs/gc.log
244/bin/rm gc-*.log
245
246#==== remove all directories that have become empty: ====
247for iteration in 1 2 3 4 5 ; do find -type d -empty -print \
248 -exec /bin/rmdir {} \; -prune; done \
249 >> .logs/empty-dirs.txt
250
251echo "========== Repo reduced: =========="
252/usr/bin/du -sc ${Repo}/*
253/usr/bin/du -sc ${Repo}/org/*
254/usr/bin/du -sc ${Repo}/org/eclipse/*
255echo "==================================="
256
257cd ${WORKSPACE}
258
259#================================================================================
260# (3) Enrich POMs
261#================================================================================
262# Add some required information to the generated poms:
263# - dynamic content (retrieved mostly from MANIFEST.MF):
264# - name
265# - url
Stephan Herrmann9a981012017-01-01 11:05:54 +0100266# - scm connection, tag and url
267# - semi dynamic
268# - developers (based on static map git-repo-base -> project leads)
Stephan Herrmannee73d532016-12-20 23:37:28 +0100269# - static content
270# - license
271# - organization
272# - issue management
273
274
275echo "==== Enrich POMs ===="
276
277cd ${Repo}
278
279echo "platform"
280java -jar ${ENRICH_POMS_JAR} `pwd`/org/eclipse/platform &> .logs/enrich-platform.txt
281echo "jdt"
282java -jar ${ENRICH_POMS_JAR} `pwd`/org/eclipse/jdt &> .logs/enrich-jdt.txt
283echo "pde"
284java -jar ${ENRICH_POMS_JAR} `pwd`/org/eclipse/pde &> .logs/enrich-pde.txt
285
286echo "== updated checksums =="
287
288function updateCheckSums() {
289 /usr/bin/md5sum ${1} | cut -d " " -f 1 > ${1}.md5
290 /usr/bin/sha1sum ${1} | cut -d " " -f 1 > ${1}.sha1
291}
292
293for pom in org/eclipse/{platform,jdt,pde}/*/*/*.pom
294do
295 updateCheckSums ${pom}
296done
297
Stephan Herrmann9a981012017-01-01 11:05:54 +0100298echo "==== Add Javadoc stubs ===="
Stephan Herrmann42f24c92016-12-31 16:53:10 +0100299
Stephan Herrmann07ae09f2017-01-01 18:58:28 +0100300cd ${Repo}
301
Stephan Herrmann42f24c92016-12-31 16:53:10 +0100302# (groupSimpleName, javadocArtifactGA)
303function createJavadocs() {
304 group=${1}
305 jar="${1}-javadoc.jar"
306 artifact=${2}
307 if [ -r ${jar} ]
308 then
309 /bin/rm ${jar}
310 fi
Stephan Herrmann9fc05c92016-12-31 17:35:26 +0100311 echo -e "Corresponding javadoc can be found in artifact ${artifact}\n" > README.txt
Stephan Herrmann42f24c92016-12-31 16:53:10 +0100312 jar cf ${jar} README.txt
313 for pom in org/eclipse/${group}/*/*/*.pom
314 do
315 javadoc=`echo ${pom} | sed -e "s|\(.*\)\.pom|\1-javadoc.jar|"`
316 /bin/cp ${jar} ${javadoc}
317 done
318}
319
320createJavadocs platform org.eclipse.platform:org.eclipse.platform.doc.isv
321createJavadocs jdt org.eclipse.jdt:org.eclipse.jdt.doc.isv
322createJavadocs pde org.eclipse.pde:org.eclipse.pde.doc.user
Stephan Herrmannee73d532016-12-20 23:37:28 +0100323
Stephan Herrmann07ae09f2017-01-01 18:58:28 +0100324echo "==== Create missing sources artifacts ===="
325
326cd ${Repo}
327
328function buildSourceJar() {
329 if [ -d assemble ]
330 then
331 /bin/rm -r assemble
332 fi
333 /bin/mkdir assemble
Stephan Herrmannc6c07eb2017-01-01 20:50:11 +0100334 giturl=file://localhost/gitroot/${1}
Stephan Herrmann07ae09f2017-01-01 18:58:28 +0100335 gitpath=${2}
336 gittag=${3}
337 group=${4}
338 artifact=${5}
339 version=${6}
340 git archive --remote=${giturl} \
341 ${gittag} ${gitpath} \
342 | tar xv
343 /bin/mv ${gitpath}/src/* assemble/
344 /bin/mv ${gitpath}/META-INF assemble/
345 if [ -d ${gitpath}/OSGI-INF ]
346 then
347 /bin/mv ${gitpath}/OSGI-INF assemble/
348 fi
349 /bin/mv ${gitpath}/about.html assemble/
Stephan Herrmannc6c07eb2017-01-01 20:50:11 +0100350 if [ $# > 6 ]
351 then
352 shift 6
353 for src in "$@"
354 do
355 /bin/mv "${gitpath}/${src}" assemble/
356 done
357 fi
Stephan Herrmann07ae09f2017-01-01 18:58:28 +0100358 /bin/rm -rf ${gitpath}
359 cd assemble
360 jar cf ../${group}/${artifact}/${version}/${artifact}-${version}-sources.jar *
361 cd -
362}
363
Stephan Herrmann4e843b02017-01-26 21:18:42 +0100364while read line
365do
366 buildSourceJar $line
367done < `dirname $0`/sourceBundles.txt
Stephan Herrmann07ae09f2017-01-01 18:58:28 +0100368
Stephan Herrmann4e843b02017-01-26 21:18:42 +0100369# special hack for missing source bundle with several source folders with blanks:
Stephan Herrmannc6c07eb2017-01-01 20:50:11 +0100370buildSourceJar platform/eclipse.platform.swt.git \
371 bundles/org.eclipse.swt.tools \
372 M20161109-0400 \
373 org/eclipse/platform org.eclipse.swt.tools 3.105.2 \
374 "JNI Generation" \
375 "Mac Generation" \
376 "Icon Exe" \
377 "NativeStats" \
378 "Mozilla Generation" \
379 "JavadocBasher"
380
Stephan Herrmannee73d532016-12-20 23:37:28 +0100381echo "========== Repo completed ========="
382
383cd ${WORKSPACE}