blob: 8f3da4801721e8104eb270d4733a8e704c971c3e [file] [log] [blame]
Stephan Herrmannee73d532016-12-20 23:37:28 +01001#!/bin/bash
2#*******************************************************************************
Stephan Herrmannb162d712018-10-27 21:37:47 +02003# Copyright (c) 2016, 2018 GK Software AG and others.
Stephan Herrmannee73d532016-12-20 23:37:28 +01004# All rights reserved. This program and the accompanying materials
5# are made available under the terms of the Eclipse Public License v1.0
6# which accompanies this distribution, and is available at
7# http://www.eclipse.org/legal/epl-v10.html
8#
9# Contributors:
10# Stephan Herrmann - initial API and implementation
11#********************************************************************************
12
13#================================================================================
14# Parameters we might want to externalize into a properties file
15#================================================================================
16
Stephan Herrmann4e843b02017-01-26 21:18:42 +010017source `dirname ${0}`/properties.sh
Stephan Herrmannee73d532016-12-20 23:37:28 +010018
Stephan Herrmannee73d532016-12-20 23:37:28 +010019
20#================================================================================
21# Util functions
22#================================================================================
23function require_executable() {
24 if [ -x ${1} ]
25 then
26 echo "Successfully installed: ${1}"
27 else
28 echo "not executable: ${1}"
29 /bin/ls -l ${1}
30 exit 1
31 fi
32}
33
Stephan Herrmann0d566b52017-10-17 17:58:14 +020034function create_baseline() {
35 cd ${Repo}
Stephan Herrmann16888a02017-10-17 22:44:37 +020036 for line in `find org/eclipse -name \*[0-9].jar | sort`
Stephan Herrmann0d566b52017-10-17 17:58:14 +020037 do
38 file=`basename $line`
Stephan Herrmann5e3f5622017-10-17 22:20:42 +020039 name=`echo $file | sed -e 's/\(.*\)-.*/\1/' | tr '.' '_'`
40 version=`echo $file | sed -e 's/.*-\(.*\)\.jar/\1/'`
Stephan Herrmannc4ae0f32017-10-17 20:54:09 +020041 previous=`eval echo \\${VERSION_$name}`
Stephan Herrmann5e3f5622017-10-17 22:20:42 +020042 if [ "$previous" != "" ]
Stephan Herrmannc4ae0f32017-10-17 20:54:09 +020043 then
44 version=${previous},${version}
45 fi
46 eval VERSION_$name=$version
Stephan Herrmann0d566b52017-10-17 17:58:14 +020047 echo VERSION_$name=$version
Stephan Herrmannce1a5e52018-07-12 21:42:12 +020048 done > ${WORKSPACE}/baseline-next.txt
Stephan Herrmann0d566b52017-10-17 17:58:14 +020049 cd -
50}
51
Stephan Herrmann9a981012017-01-01 11:05:54 +010052#================================================================================
53# (1) Install and run the CBI aggregator
54#================================================================================
55echo "==== CBI aggregator ===="
56
Stephan Herrmannee73d532016-12-20 23:37:28 +010057if [ ! -d ${LOCAL_TOOLS} ]
58then
59 /bin/mkdir ${LOCAL_TOOLS}
60fi
61
62if [ ! -x ${ECLIPSE} ]
63then
64 cd ${LOCAL_TOOLS}
Stephan Herrmann9a981012017-01-01 11:05:54 +010065 echo "Extracting Eclipse from ${FILE_ECLIPSE} ..."
Stephan Herrmannee73d532016-12-20 23:37:28 +010066 tar xf ${FILE_ECLIPSE}
67 cd ${WORKSPACE}
68fi
69require_executable ${ECLIPSE}
70
71if [ ! -x ${AGGREGATOR} ]
72then
Stephan Herrmann9a981012017-01-01 11:05:54 +010073 echo "Installing the CBI aggregator into ${LOCAL_TOOLS}/${DIR_AGGREGATOR} ..."
Stephan Herrmannee73d532016-12-20 23:37:28 +010074 ${ECLIPSE} -application ${APP_NAME_P2DIRECTOR} \
75 -r ${URL_AGG_UPDATES} \
76 -d ${LOCAL_TOOLS}/${DIR_AGGREGATOR} -p CBIProfile \
77 -installIU ${IU_AGG_PRODUCT}
78fi
79require_executable ${AGGREGATOR}
80
81RepoRaw=${WORKSPACE}/reporaw-${BUILD_NUMBER}
82Repo=${WORKSPACE}/repo-${BUILD_NUMBER}
83/bin/mkdir ${RepoRaw}
84
Stephan Herrmann9a981012017-01-01 11:05:54 +010085echo "Running the aggregator with build model ${FILE_SDK_AGGR} ..."
Stephan Herrmannee73d532016-12-20 23:37:28 +010086${AGGREGATOR} aggregate --buildModel ${FILE_SDK_AGGR} --action CLEAN_BUILD --buildRoot ${RepoRaw}
87if [ "$?" != "0" ]
88then
89 echo "FAILURE $?"
90 exit 1
91fi
92/bin/mv ${RepoRaw}/final ${Repo}
93/bin/rm -rf ${RepoRaw}
94
95echo "========== Repo created: =========="
96/usr/bin/du -sc ${Repo}/*
97/usr/bin/du -sc ${Repo}/org/*
98/usr/bin/du -sc ${Repo}/org/eclipse/*
99echo "==================================="
100
101
102#================================================================================
103# (2) Remove irrelevant stuff
104#================================================================================
105# Removes from the build output of cbiAggregator everything that is not relevant for maven.
106# All removed directories / files will be logged to .logs/removed.txt
107
108echo "==== Remove irrelevant stuff ===="
109
110cd ${Repo}
111
112if [ ! -d .logs ]
113then
114 /bin/mkdir .logs
115elif [ -f .logs/removed.txt ]
116then
117 /bin/rm .logs/removed.txt
118fi
119
120#==== remove the p2 repository (not logged): ====
121
122/bin/rm -r p2.index p2.packed content.jar artifacts.jar
123
Stephan Herrmannee73d532016-12-20 23:37:28 +0100124#==== remove features: ====
125
126echo "== Features: ==" | tee >> .logs/removed.txt
127
128/usr/bin/find * -type d -name \*feature.group -print -exec /bin/rm -rf {} \; -prune >> .logs/removed.txt
129/usr/bin/find * -type d -name \*feature.jar -print -exec /bin/rm -rf {} \; -prune >> .logs/removed.txt
130
131#==== remove eclipse test plug-ins: ====
132
133echo "== Test plugins: ==" | tee >> .logs/removed.txt
134
135ls -d org/eclipse/*/*test* >> .logs/removed.txt
136/bin/rm -r org/eclipse/*/*test*
137
138#==== remove other non-artifacts: ====
139
140echo "== Other non-artifacts: ==" | tee >> .logs/removed.txt
141
142/usr/bin/find tooling -type d >> .logs/removed.txt
143/bin/rm -r tooling*
144
145# ... folders that contain only 1.2.3/foo-1.2.3.pom but no corresponding 1.2.3/foo-1.2.3.jar:
146function hasPomButNoJar() {
147 cd ${1}
148 # expect only one sub-directory, starting with a digit, plus maven-metadata.xml*:
149 other=`ls -d [!0-9]* 2> /dev/null`
Stephan Herrmann0b6f3932017-01-01 21:17:57 +0100150 if echo "${other}" | tr "\n" " " | egrep "^maven-metadata.xml maven-metadata.xml.md5 maven-metadata.xml.sha1 \$"
Stephan Herrmannee73d532016-12-20 23:37:28 +0100151 then
Stephan Herrmann0a358922017-01-01 11:19:36 +0100152 : # clean -> proceed below
Stephan Herrmann9a981012017-01-01 11:05:54 +0100153 else
154 exit 1 # unexpected content found, don't remove
Stephan Herrmannee73d532016-12-20 23:37:28 +0100155 fi
156 # scan all *.pom inside the version sub-directory
157 r=1
158 for pom in `ls [0-9]*/*.pom 2> /dev/null`
159 do
160 jar=`echo ${pom} | sed -e "s|\(.*\)\.pom|\1.jar|"`
161 if [ -f ${jar} ]
162 then
163 # jar found, so keep it
164 exit 1
165 fi
166 # pom without jar found, let's answer true below
167 r=0
168 done
169 exit $r
170}
171export -f hasPomButNoJar
172
173/usr/bin/find org/eclipse/{jdt,pde,platform} -type d \
174 -exec /bin/bash -c 'hasPomButNoJar "$@"' bash {} \; \
175 -print -exec /bin/rm -rf {} \; -prune >> .logs/removed.txt
176# second "bash" is used as $0 in the function
177
178cd ${WORKSPACE}
179
180echo "========== Repo reduced: =========="
181/usr/bin/du -sc ${Repo}/*
182/usr/bin/du -sc ${Repo}/org/*
183/usr/bin/du -sc ${Repo}/org/eclipse/*
184echo "==================================="
185
186#================================================================================
187# (2) Garbage Collector
188#================================================================================
189# Removes from the build output of cbiAggregator everything that is not referenced
190# from any pom below org/eclipse/{platform,jdt,pde}
191#
192# Log output:
193# .logs/removedGarbage.txt all directories during garbage collection
194# .logs/gc.log incoming dependencies of retained artifacts
195# .logs/empty-dirs.txt removed empty directories
196
197echo "==== Garbage Collector ===="
198
199cd ${Repo}
200
201#==== function gc_bundle(): ====
202# Test if pom ${1} is referenced in any other pom.
203# If not, append the containing directory to the file "toremove.txt"
204function gc_bundle {
205 AID=`echo ${1} | sed -e "s|.*/\(.*\)[_-].*|\1|"`
206 DIR=`echo ${1} | sed -e "s|\(.*\)/[0-9].*|\1|"`
207 POM=`basename ${1}`
208
209 ANSWER=`find org/eclipse/{platform,jdt,pde} -name \*.pom \! -name ${POM} \
210 -exec /bin/grep -q "<artifactId>${AID}</artifactId>" {} \; -print -quit`
211
212 if [ "$ANSWER" == "" ]
213 then
214 echo "Will remove $DIR"
215 echo $DIR >> toremove.txt
216 else
217 echo "$1 is used by $ANSWER"
218 fi
219}
220export -f gc_bundle
221
222#==== run the garbage collector: ====
223# iterate (max 5 times) in case artifacts were used only from garbage:
224for iteration in 1 2 3 4 5
225do
226 echo "== GC iteration ${iteration} =="
227
228 # look for garbage only outside platform, jdt or pde folders:
229 find -name platform -prune -o -name jdt -prune -o -name pde -prune -o \
230 -name \*.pom -exec /bin/bash -c 'gc_bundle "$@"' bash {} \; \
231 > gc-${iteration}.log
232 # second "bash" is used as $0 in the function
233
234 if [ ! -f toremove.txt ]
235 then
236 # no more garbage found
237 break
238 fi
239 cat toremove.txt >> .logs/removedGarbage.txt
240 for d in `cat toremove.txt`; do /bin/rm -r $d; done
241 /bin/rm toremove.txt
242done
243
244# merge gc logs:
245cat gc-*.log | sort --unique > .logs/gc.log
246/bin/rm gc-*.log
247
248#==== remove all directories that have become empty: ====
Stephan Herrmann3b557672017-01-26 22:26:12 +0100249for iteration in 1 2 3 4 5 6 ; do find -type d -empty -print \
Stephan Herrmannee73d532016-12-20 23:37:28 +0100250 -exec /bin/rmdir {} \; -prune; done \
251 >> .logs/empty-dirs.txt
252
253echo "========== Repo reduced: =========="
254/usr/bin/du -sc ${Repo}/*
255/usr/bin/du -sc ${Repo}/org/*
256/usr/bin/du -sc ${Repo}/org/eclipse/*
257echo "==================================="
258
259cd ${WORKSPACE}
260
261#================================================================================
262# (3) Enrich POMs
263#================================================================================
264# Add some required information to the generated poms:
265# - dynamic content (retrieved mostly from MANIFEST.MF):
266# - name
267# - url
Stephan Herrmann9a981012017-01-01 11:05:54 +0100268# - scm connection, tag and url
269# - semi dynamic
270# - developers (based on static map git-repo-base -> project leads)
Stephan Herrmannee73d532016-12-20 23:37:28 +0100271# - static content
272# - license
273# - organization
274# - issue management
275
276
277echo "==== Enrich POMs ===="
278
Stephan Herrmannb162d712018-10-27 21:37:47 +0200279# build the jar:
280cd ${WORKSPACE}/work
281mkdir bin
282cd src
283javac -d ../bin/ `find * -name \*.java`
284cd ../bin
285jar cefv ${ENRICH_POMS_PACKAGE}.EnrichPoms ${ENRICH_POMS_JAR} `find * -name \*.class`
286cd ${WORKSPACE}
287/bin/ls -l ${ENRICH_POMS_JAR}
288
Stephan Herrmannee73d532016-12-20 23:37:28 +0100289cd ${Repo}
290
291echo "platform"
292java -jar ${ENRICH_POMS_JAR} `pwd`/org/eclipse/platform &> .logs/enrich-platform.txt
293echo "jdt"
294java -jar ${ENRICH_POMS_JAR} `pwd`/org/eclipse/jdt &> .logs/enrich-jdt.txt
295echo "pde"
296java -jar ${ENRICH_POMS_JAR} `pwd`/org/eclipse/pde &> .logs/enrich-pde.txt
297
Stephan Herrmannee73d532016-12-20 23:37:28 +0100298
Stephan Herrmann9a981012017-01-01 11:05:54 +0100299echo "==== Add Javadoc stubs ===="
Stephan Herrmann42f24c92016-12-31 16:53:10 +0100300
Stephan Herrmann07ae09f2017-01-01 18:58:28 +0100301cd ${Repo}
302
Stephan Herrmann42f24c92016-12-31 16:53:10 +0100303# (groupSimpleName, javadocArtifactGA)
304function createJavadocs() {
305 group=${1}
306 jar="${1}-javadoc.jar"
307 artifact=${2}
308 if [ -r ${jar} ]
309 then
310 /bin/rm ${jar}
311 fi
Stephan Herrmann9fc05c92016-12-31 17:35:26 +0100312 echo -e "Corresponding javadoc can be found in artifact ${artifact}\n" > README.txt
Stephan Herrmann42f24c92016-12-31 16:53:10 +0100313 jar cf ${jar} README.txt
314 for pom in org/eclipse/${group}/*/*/*.pom
315 do
316 javadoc=`echo ${pom} | sed -e "s|\(.*\)\.pom|\1-javadoc.jar|"`
317 /bin/cp ${jar} ${javadoc}
318 done
319}
320
321createJavadocs platform org.eclipse.platform:org.eclipse.platform.doc.isv
322createJavadocs jdt org.eclipse.jdt:org.eclipse.jdt.doc.isv
323createJavadocs pde org.eclipse.pde:org.eclipse.pde.doc.user
Stephan Herrmannee73d532016-12-20 23:37:28 +0100324
Stephan Herrmann07ae09f2017-01-01 18:58:28 +0100325echo "==== Create missing sources artifacts ===="
326
327cd ${Repo}
328
329function buildSourceJar() {
330 if [ -d assemble ]
331 then
332 /bin/rm -r assemble
333 fi
334 /bin/mkdir assemble
Stephan Herrmannc6c07eb2017-01-01 20:50:11 +0100335 giturl=file://localhost/gitroot/${1}
Stephan Herrmann07ae09f2017-01-01 18:58:28 +0100336 gitpath=${2}
337 gittag=${3}
338 group=${4}
339 artifact=${5}
340 version=${6}
341 git archive --remote=${giturl} \
342 ${gittag} ${gitpath} \
343 | tar xv
344 /bin/mv ${gitpath}/src/* assemble/
345 /bin/mv ${gitpath}/META-INF assemble/
346 if [ -d ${gitpath}/OSGI-INF ]
347 then
348 /bin/mv ${gitpath}/OSGI-INF assemble/
349 fi
350 /bin/mv ${gitpath}/about.html assemble/
Stephan Herrmannc6c07eb2017-01-01 20:50:11 +0100351 if [ $# > 6 ]
352 then
353 shift 6
354 for src in "$@"
355 do
356 /bin/mv "${gitpath}/${src}" assemble/
357 done
358 fi
Stephan Herrmann07ae09f2017-01-01 18:58:28 +0100359 /bin/rm -rf ${gitpath}
360 cd assemble
361 jar cf ../${group}/${artifact}/${version}/${artifact}-${version}-sources.jar *
362 cd -
363}
364
Stephan Herrmann4e843b02017-01-26 21:18:42 +0100365while read line
366do
367 buildSourceJar $line
Stephan Herrmann0def6d52018-10-27 23:01:53 +0200368done < ${WORKSPACE}/work/sourceBundles.txt
Stephan Herrmann07ae09f2017-01-01 18:58:28 +0100369
Stephan Herrmann95fbf622017-03-05 13:43:09 +0100370# copy ecj-src from build output (NB: we are mapping from SDK version (4.x) back to ECJ version (3.x)):
Stephan Herrmann9fb254e2017-03-05 13:53:22 +0100371/bin/cp ${DROPS4}/${SDK_BUILD_DIR}/ecjsrc-${SDK_VERSION}.jar org/eclipse/jdt/ecj/${ECJ_VERSION}/ecj-${ECJ_VERSION}-sources.jar
Stephan Herrmann95fbf622017-03-05 13:43:09 +0100372
Stephan Herrmannee73d532016-12-20 23:37:28 +0100373echo "========== Repo completed ========="
374
Stephan Herrmann0d566b52017-10-17 17:58:14 +0200375create_baseline
376
377echo "========== Next baseline created ========="
378
Stephan Herrmannee73d532016-12-20 23:37:28 +0100379cd ${WORKSPACE}