blob: 14a1e91521cbc9dedb893838a1ea746183677054 [file] [log] [blame]
Stephan Herrmannee73d532016-12-20 23:37:28 +01001#!/bin/bash
2#*******************************************************************************
3# Copyright (c) 2016 GK Software AG and others.
4# All rights reserved. This program and the accompanying materials
5# are made available under the terms of the Eclipse Public License v1.0
6# which accompanies this distribution, and is available at
7# http://www.eclipse.org/legal/epl-v10.html
8#
9# Contributors:
10# Stephan Herrmann - initial API and implementation
11#********************************************************************************
12
13#================================================================================
14# Parameters we might want to externalize into a properties file
15#================================================================================
16
17# ECLIPSE:
18DROPS4=/home/data/httpd/archive.eclipse.org/eclipse/downloads/drops4
19FILE_ECLIPSE=${DROPS4}/R-4.6.1-201609071200/eclipse-SDK-4.6.1-linux-gtk-x86_64.tar.gz
20APP_NAME_P2DIRECTOR=org.eclipse.equinox.p2.director
21
22# QUESTION: set tmpdir? (-vmargs not accepted by director application?)
23
24# AGGREGATOR:
25IU_AGG_PRODUCT=org.eclipse.cbi.p2repo.cli.product
26URL_AGG_UPDATES=http://download.eclipse.org/cbi/updates/aggregator/headless/4.6/
27
Stephan Herrmannb3148c62016-12-21 00:42:13 +010028FILE_SDK_AGGR=${WORKSPACE}/SDK4Mvn.aggr
Stephan Herrmannee73d532016-12-20 23:37:28 +010029
30# ENRICH POMS tool:
Stephan Herrmann8f2b5ab2016-12-21 00:23:47 +010031ENRICH_POMS_JAR=${WORKSPACE}/../../pomEnricher/workspace/EnrichPoms.jar
Stephan Herrmannee73d532016-12-20 23:37:28 +010032
33# LOCAL TOOLS:
34LOCAL_TOOLS=${WORKSPACE}/tools
35DIR_AGGREGATOR=aggregator
36AGGREGATOR=${LOCAL_TOOLS}/${DIR_AGGREGATOR}/cbiAggr
37ECLIPSE=${LOCAL_TOOLS}/eclipse/eclipse
38
39#================================================================================
40# Util functions
41#================================================================================
42function require_executable() {
43 if [ -x ${1} ]
44 then
45 echo "Successfully installed: ${1}"
46 else
47 echo "not executable: ${1}"
48 /bin/ls -l ${1}
49 exit 1
50 fi
51}
52
Stephan Herrmann9a981012017-01-01 11:05:54 +010053#================================================================================
54# (1) Install and run the CBI aggregator
55#================================================================================
56echo "==== CBI aggregator ===="
57
Stephan Herrmannb3148c62016-12-21 00:42:13 +010058# -------- fetch .aggr file **TEMP** will eventually move to the releng git: ------------
59git archive --remote=file://localhost/gitroot/objectteams/org.eclipse.objectteams.git \
60 master releng/EnrichPoms/SDK4Mvn.aggr \
61 | tar xv
62/bin/mv releng/EnrichPoms/SDK4Mvn.aggr ${WORKSPACE}/
63/bin/rmdir -p releng/EnrichPoms
64
Stephan Herrmannee73d532016-12-20 23:37:28 +010065if [ ! -d ${LOCAL_TOOLS} ]
66then
67 /bin/mkdir ${LOCAL_TOOLS}
68fi
69
70if [ ! -x ${ECLIPSE} ]
71then
72 cd ${LOCAL_TOOLS}
Stephan Herrmann9a981012017-01-01 11:05:54 +010073 echo "Extracting Eclipse from ${FILE_ECLIPSE} ..."
Stephan Herrmannee73d532016-12-20 23:37:28 +010074 tar xf ${FILE_ECLIPSE}
75 cd ${WORKSPACE}
76fi
77require_executable ${ECLIPSE}
78
79if [ ! -x ${AGGREGATOR} ]
80then
Stephan Herrmann9a981012017-01-01 11:05:54 +010081 echo "Installing the CBI aggregator into ${LOCAL_TOOLS}/${DIR_AGGREGATOR} ..."
Stephan Herrmannee73d532016-12-20 23:37:28 +010082 ${ECLIPSE} -application ${APP_NAME_P2DIRECTOR} \
83 -r ${URL_AGG_UPDATES} \
84 -d ${LOCAL_TOOLS}/${DIR_AGGREGATOR} -p CBIProfile \
85 -installIU ${IU_AGG_PRODUCT}
86fi
87require_executable ${AGGREGATOR}
88
89RepoRaw=${WORKSPACE}/reporaw-${BUILD_NUMBER}
90Repo=${WORKSPACE}/repo-${BUILD_NUMBER}
91/bin/mkdir ${RepoRaw}
92
Stephan Herrmann9a981012017-01-01 11:05:54 +010093echo "Running the aggregator with build model ${FILE_SDK_AGGR} ..."
Stephan Herrmannee73d532016-12-20 23:37:28 +010094${AGGREGATOR} aggregate --buildModel ${FILE_SDK_AGGR} --action CLEAN_BUILD --buildRoot ${RepoRaw}
95if [ "$?" != "0" ]
96then
97 echo "FAILURE $?"
98 exit 1
99fi
100/bin/mv ${RepoRaw}/final ${Repo}
101/bin/rm -rf ${RepoRaw}
102
103echo "========== Repo created: =========="
104/usr/bin/du -sc ${Repo}/*
105/usr/bin/du -sc ${Repo}/org/*
106/usr/bin/du -sc ${Repo}/org/eclipse/*
107echo "==================================="
108
109
110#================================================================================
111# (2) Remove irrelevant stuff
112#================================================================================
113# Removes from the build output of cbiAggregator everything that is not relevant for maven.
114# All removed directories / files will be logged to .logs/removed.txt
115
116echo "==== Remove irrelevant stuff ===="
117
118cd ${Repo}
119
120if [ ! -d .logs ]
121then
122 /bin/mkdir .logs
123elif [ -f .logs/removed.txt ]
124then
125 /bin/rm .logs/removed.txt
126fi
127
128#==== remove the p2 repository (not logged): ====
129
130/bin/rm -r p2.index p2.packed content.jar artifacts.jar
131
132#==== remove -sources artifacts, misplaced due to quirk from https://bugs.eclipse.org/508910: ====
133
134echo "== Misplaced -sources artifacts: ==" | tee >> .logs/removed.txt
135
136# works only outside org/eclipse/{platform,jdt,pde}:
137
138/usr/bin/find -type d -name platform -prune -o -name jdt -prune -o -name pde -prune -o \
139 -name \*-sources.jar\* -print -exec /bin/rm {} \; >> .logs/removed.txt
140
141#==== remove features: ====
142
143echo "== Features: ==" | tee >> .logs/removed.txt
144
145/usr/bin/find * -type d -name \*feature.group -print -exec /bin/rm -rf {} \; -prune >> .logs/removed.txt
146/usr/bin/find * -type d -name \*feature.jar -print -exec /bin/rm -rf {} \; -prune >> .logs/removed.txt
147
148#==== remove eclipse test plug-ins: ====
149
150echo "== Test plugins: ==" | tee >> .logs/removed.txt
151
152ls -d org/eclipse/*/*test* >> .logs/removed.txt
153/bin/rm -r org/eclipse/*/*test*
154
155#==== remove other non-artifacts: ====
156
157echo "== Other non-artifacts: ==" | tee >> .logs/removed.txt
158
159/usr/bin/find tooling -type d >> .logs/removed.txt
160/bin/rm -r tooling*
161
162# ... folders that contain only 1.2.3/foo-1.2.3.pom but no corresponding 1.2.3/foo-1.2.3.jar:
163function hasPomButNoJar() {
164 cd ${1}
165 # expect only one sub-directory, starting with a digit, plus maven-metadata.xml*:
166 other=`ls -d [!0-9]* 2> /dev/null`
Stephan Herrmann9a981012017-01-01 11:05:54 +0100167 if `echo "${other}" | tr "\n" " " | egrep "^maven-metadata.xml maven-metadata.xml.md5 maven-metadata.xml.sha1 \$"`
Stephan Herrmannee73d532016-12-20 23:37:28 +0100168 then
Stephan Herrmann0a358922017-01-01 11:19:36 +0100169 : # clean -> proceed below
Stephan Herrmann9a981012017-01-01 11:05:54 +0100170 else
171 exit 1 # unexpected content found, don't remove
Stephan Herrmannee73d532016-12-20 23:37:28 +0100172 fi
173 # scan all *.pom inside the version sub-directory
174 r=1
175 for pom in `ls [0-9]*/*.pom 2> /dev/null`
176 do
177 jar=`echo ${pom} | sed -e "s|\(.*\)\.pom|\1.jar|"`
178 if [ -f ${jar} ]
179 then
180 # jar found, so keep it
181 exit 1
182 fi
183 # pom without jar found, let's answer true below
184 r=0
185 done
186 exit $r
187}
188export -f hasPomButNoJar
189
190/usr/bin/find org/eclipse/{jdt,pde,platform} -type d \
191 -exec /bin/bash -c 'hasPomButNoJar "$@"' bash {} \; \
192 -print -exec /bin/rm -rf {} \; -prune >> .logs/removed.txt
193# second "bash" is used as $0 in the function
194
195cd ${WORKSPACE}
196
197echo "========== Repo reduced: =========="
198/usr/bin/du -sc ${Repo}/*
199/usr/bin/du -sc ${Repo}/org/*
200/usr/bin/du -sc ${Repo}/org/eclipse/*
201echo "==================================="
202
203#================================================================================
204# (2) Garbage Collector
205#================================================================================
206# Removes from the build output of cbiAggregator everything that is not referenced
207# from any pom below org/eclipse/{platform,jdt,pde}
208#
209# Log output:
210# .logs/removedGarbage.txt all directories during garbage collection
211# .logs/gc.log incoming dependencies of retained artifacts
212# .logs/empty-dirs.txt removed empty directories
213
214echo "==== Garbage Collector ===="
215
216cd ${Repo}
217
218#==== function gc_bundle(): ====
219# Test if pom ${1} is referenced in any other pom.
220# If not, append the containing directory to the file "toremove.txt"
221function gc_bundle {
222 AID=`echo ${1} | sed -e "s|.*/\(.*\)[_-].*|\1|"`
223 DIR=`echo ${1} | sed -e "s|\(.*\)/[0-9].*|\1|"`
224 POM=`basename ${1}`
225
226 ANSWER=`find org/eclipse/{platform,jdt,pde} -name \*.pom \! -name ${POM} \
227 -exec /bin/grep -q "<artifactId>${AID}</artifactId>" {} \; -print -quit`
228
229 if [ "$ANSWER" == "" ]
230 then
231 echo "Will remove $DIR"
232 echo $DIR >> toremove.txt
233 else
234 echo "$1 is used by $ANSWER"
235 fi
236}
237export -f gc_bundle
238
239#==== run the garbage collector: ====
240# iterate (max 5 times) in case artifacts were used only from garbage:
241for iteration in 1 2 3 4 5
242do
243 echo "== GC iteration ${iteration} =="
244
245 # look for garbage only outside platform, jdt or pde folders:
246 find -name platform -prune -o -name jdt -prune -o -name pde -prune -o \
247 -name \*.pom -exec /bin/bash -c 'gc_bundle "$@"' bash {} \; \
248 > gc-${iteration}.log
249 # second "bash" is used as $0 in the function
250
251 if [ ! -f toremove.txt ]
252 then
253 # no more garbage found
254 break
255 fi
256 cat toremove.txt >> .logs/removedGarbage.txt
257 for d in `cat toremove.txt`; do /bin/rm -r $d; done
258 /bin/rm toremove.txt
259done
260
261# merge gc logs:
262cat gc-*.log | sort --unique > .logs/gc.log
263/bin/rm gc-*.log
264
265#==== remove all directories that have become empty: ====
266for iteration in 1 2 3 4 5 ; do find -type d -empty -print \
267 -exec /bin/rmdir {} \; -prune; done \
268 >> .logs/empty-dirs.txt
269
270echo "========== Repo reduced: =========="
271/usr/bin/du -sc ${Repo}/*
272/usr/bin/du -sc ${Repo}/org/*
273/usr/bin/du -sc ${Repo}/org/eclipse/*
274echo "==================================="
275
276cd ${WORKSPACE}
277
278#================================================================================
279# (3) Enrich POMs
280#================================================================================
281# Add some required information to the generated poms:
282# - dynamic content (retrieved mostly from MANIFEST.MF):
283# - name
284# - url
Stephan Herrmann9a981012017-01-01 11:05:54 +0100285# - scm connection, tag and url
286# - semi dynamic
287# - developers (based on static map git-repo-base -> project leads)
Stephan Herrmannee73d532016-12-20 23:37:28 +0100288# - static content
289# - license
290# - organization
291# - issue management
292
293
294echo "==== Enrich POMs ===="
295
296cd ${Repo}
297
298echo "platform"
299java -jar ${ENRICH_POMS_JAR} `pwd`/org/eclipse/platform &> .logs/enrich-platform.txt
300echo "jdt"
301java -jar ${ENRICH_POMS_JAR} `pwd`/org/eclipse/jdt &> .logs/enrich-jdt.txt
302echo "pde"
303java -jar ${ENRICH_POMS_JAR} `pwd`/org/eclipse/pde &> .logs/enrich-pde.txt
304
305echo "== updated checksums =="
306
307function updateCheckSums() {
308 /usr/bin/md5sum ${1} | cut -d " " -f 1 > ${1}.md5
309 /usr/bin/sha1sum ${1} | cut -d " " -f 1 > ${1}.sha1
310}
311
312for pom in org/eclipse/{platform,jdt,pde}/*/*/*.pom
313do
314 updateCheckSums ${pom}
315done
316
Stephan Herrmann9a981012017-01-01 11:05:54 +0100317echo "==== Add Javadoc stubs ===="
Stephan Herrmann42f24c92016-12-31 16:53:10 +0100318
319# (groupSimpleName, javadocArtifactGA)
320function createJavadocs() {
321 group=${1}
322 jar="${1}-javadoc.jar"
323 artifact=${2}
324 if [ -r ${jar} ]
325 then
326 /bin/rm ${jar}
327 fi
Stephan Herrmann9fc05c92016-12-31 17:35:26 +0100328 echo -e "Corresponding javadoc can be found in artifact ${artifact}\n" > README.txt
Stephan Herrmann42f24c92016-12-31 16:53:10 +0100329 jar cf ${jar} README.txt
330 for pom in org/eclipse/${group}/*/*/*.pom
331 do
332 javadoc=`echo ${pom} | sed -e "s|\(.*\)\.pom|\1-javadoc.jar|"`
333 /bin/cp ${jar} ${javadoc}
334 done
335}
336
337createJavadocs platform org.eclipse.platform:org.eclipse.platform.doc.isv
338createJavadocs jdt org.eclipse.jdt:org.eclipse.jdt.doc.isv
339createJavadocs pde org.eclipse.pde:org.eclipse.pde.doc.user
Stephan Herrmannee73d532016-12-20 23:37:28 +0100340
341echo "========== Repo completed ========="
342
343cd ${WORKSPACE}