Stephan Herrmann | ee73d53 | 2016-12-20 23:37:28 +0100 | [diff] [blame] | 1 | #!/bin/bash |
| 2 | #******************************************************************************* |
| 3 | # Copyright (c) 2016 GK Software AG and others. |
| 4 | # All rights reserved. This program and the accompanying materials |
| 5 | # are made available under the terms of the Eclipse Public License v1.0 |
| 6 | # which accompanies this distribution, and is available at |
| 7 | # http://www.eclipse.org/legal/epl-v10.html |
| 8 | # |
| 9 | # Contributors: |
| 10 | # Stephan Herrmann - initial API and implementation |
| 11 | #******************************************************************************** |
| 12 | |
| 13 | #================================================================================ |
| 14 | # Parameters we might want to externalize into a properties file |
| 15 | #================================================================================ |
| 16 | |
Stephan Herrmann | 4e843b0 | 2017-01-26 21:18:42 +0100 | [diff] [blame^] | 17 | source `dirname ${0}`/properties.sh |
Stephan Herrmann | ee73d53 | 2016-12-20 23:37:28 +0100 | [diff] [blame] | 18 | |
Stephan Herrmann | ee73d53 | 2016-12-20 23:37:28 +0100 | [diff] [blame] | 19 | |
| 20 | #================================================================================ |
| 21 | # Util functions |
| 22 | #================================================================================ |
| 23 | function require_executable() { |
| 24 | if [ -x ${1} ] |
| 25 | then |
| 26 | echo "Successfully installed: ${1}" |
| 27 | else |
| 28 | echo "not executable: ${1}" |
| 29 | /bin/ls -l ${1} |
| 30 | exit 1 |
| 31 | fi |
| 32 | } |
| 33 | |
Stephan Herrmann | 9a98101 | 2017-01-01 11:05:54 +0100 | [diff] [blame] | 34 | #================================================================================ |
| 35 | # (1) Install and run the CBI aggregator |
| 36 | #================================================================================ |
| 37 | echo "==== CBI aggregator ====" |
| 38 | |
Stephan Herrmann | b3148c6 | 2016-12-21 00:42:13 +0100 | [diff] [blame] | 39 | # -------- fetch .aggr file **TEMP** will eventually move to the releng git: ------------ |
| 40 | git archive --remote=file://localhost/gitroot/objectteams/org.eclipse.objectteams.git \ |
| 41 | master releng/EnrichPoms/SDK4Mvn.aggr \ |
| 42 | | tar xv |
| 43 | /bin/mv releng/EnrichPoms/SDK4Mvn.aggr ${WORKSPACE}/ |
| 44 | /bin/rmdir -p releng/EnrichPoms |
| 45 | |
Stephan Herrmann | ee73d53 | 2016-12-20 23:37:28 +0100 | [diff] [blame] | 46 | if [ ! -d ${LOCAL_TOOLS} ] |
| 47 | then |
| 48 | /bin/mkdir ${LOCAL_TOOLS} |
| 49 | fi |
| 50 | |
| 51 | if [ ! -x ${ECLIPSE} ] |
| 52 | then |
| 53 | cd ${LOCAL_TOOLS} |
Stephan Herrmann | 9a98101 | 2017-01-01 11:05:54 +0100 | [diff] [blame] | 54 | echo "Extracting Eclipse from ${FILE_ECLIPSE} ..." |
Stephan Herrmann | ee73d53 | 2016-12-20 23:37:28 +0100 | [diff] [blame] | 55 | tar xf ${FILE_ECLIPSE} |
| 56 | cd ${WORKSPACE} |
| 57 | fi |
| 58 | require_executable ${ECLIPSE} |
| 59 | |
| 60 | if [ ! -x ${AGGREGATOR} ] |
| 61 | then |
Stephan Herrmann | 9a98101 | 2017-01-01 11:05:54 +0100 | [diff] [blame] | 62 | echo "Installing the CBI aggregator into ${LOCAL_TOOLS}/${DIR_AGGREGATOR} ..." |
Stephan Herrmann | ee73d53 | 2016-12-20 23:37:28 +0100 | [diff] [blame] | 63 | ${ECLIPSE} -application ${APP_NAME_P2DIRECTOR} \ |
| 64 | -r ${URL_AGG_UPDATES} \ |
| 65 | -d ${LOCAL_TOOLS}/${DIR_AGGREGATOR} -p CBIProfile \ |
| 66 | -installIU ${IU_AGG_PRODUCT} |
| 67 | fi |
| 68 | require_executable ${AGGREGATOR} |
| 69 | |
| 70 | RepoRaw=${WORKSPACE}/reporaw-${BUILD_NUMBER} |
| 71 | Repo=${WORKSPACE}/repo-${BUILD_NUMBER} |
| 72 | /bin/mkdir ${RepoRaw} |
| 73 | |
Stephan Herrmann | 9a98101 | 2017-01-01 11:05:54 +0100 | [diff] [blame] | 74 | echo "Running the aggregator with build model ${FILE_SDK_AGGR} ..." |
Stephan Herrmann | ee73d53 | 2016-12-20 23:37:28 +0100 | [diff] [blame] | 75 | ${AGGREGATOR} aggregate --buildModel ${FILE_SDK_AGGR} --action CLEAN_BUILD --buildRoot ${RepoRaw} |
| 76 | if [ "$?" != "0" ] |
| 77 | then |
| 78 | echo "FAILURE $?" |
| 79 | exit 1 |
| 80 | fi |
| 81 | /bin/mv ${RepoRaw}/final ${Repo} |
| 82 | /bin/rm -rf ${RepoRaw} |
| 83 | |
| 84 | echo "========== Repo created: ==========" |
| 85 | /usr/bin/du -sc ${Repo}/* |
| 86 | /usr/bin/du -sc ${Repo}/org/* |
| 87 | /usr/bin/du -sc ${Repo}/org/eclipse/* |
| 88 | echo "===================================" |
| 89 | |
| 90 | |
| 91 | #================================================================================ |
| 92 | # (2) Remove irrelevant stuff |
| 93 | #================================================================================ |
| 94 | # Removes from the build output of cbiAggregator everything that is not relevant for maven. |
| 95 | # All removed directories / files will be logged to .logs/removed.txt |
| 96 | |
| 97 | echo "==== Remove irrelevant stuff ====" |
| 98 | |
| 99 | cd ${Repo} |
| 100 | |
| 101 | if [ ! -d .logs ] |
| 102 | then |
| 103 | /bin/mkdir .logs |
| 104 | elif [ -f .logs/removed.txt ] |
| 105 | then |
| 106 | /bin/rm .logs/removed.txt |
| 107 | fi |
| 108 | |
| 109 | #==== remove the p2 repository (not logged): ==== |
| 110 | |
| 111 | /bin/rm -r p2.index p2.packed content.jar artifacts.jar |
| 112 | |
| 113 | #==== remove -sources artifacts, misplaced due to quirk from https://bugs.eclipse.org/508910: ==== |
| 114 | |
| 115 | echo "== Misplaced -sources artifacts: ==" | tee >> .logs/removed.txt |
| 116 | |
| 117 | # works only outside org/eclipse/{platform,jdt,pde}: |
| 118 | |
| 119 | /usr/bin/find -type d -name platform -prune -o -name jdt -prune -o -name pde -prune -o \ |
| 120 | -name \*-sources.jar\* -print -exec /bin/rm {} \; >> .logs/removed.txt |
| 121 | |
| 122 | #==== remove features: ==== |
| 123 | |
| 124 | echo "== Features: ==" | tee >> .logs/removed.txt |
| 125 | |
| 126 | /usr/bin/find * -type d -name \*feature.group -print -exec /bin/rm -rf {} \; -prune >> .logs/removed.txt |
| 127 | /usr/bin/find * -type d -name \*feature.jar -print -exec /bin/rm -rf {} \; -prune >> .logs/removed.txt |
| 128 | |
| 129 | #==== remove eclipse test plug-ins: ==== |
| 130 | |
| 131 | echo "== Test plugins: ==" | tee >> .logs/removed.txt |
| 132 | |
| 133 | ls -d org/eclipse/*/*test* >> .logs/removed.txt |
| 134 | /bin/rm -r org/eclipse/*/*test* |
| 135 | |
| 136 | #==== remove other non-artifacts: ==== |
| 137 | |
| 138 | echo "== Other non-artifacts: ==" | tee >> .logs/removed.txt |
| 139 | |
| 140 | /usr/bin/find tooling -type d >> .logs/removed.txt |
| 141 | /bin/rm -r tooling* |
| 142 | |
| 143 | # ... folders that contain only 1.2.3/foo-1.2.3.pom but no corresponding 1.2.3/foo-1.2.3.jar: |
| 144 | function hasPomButNoJar() { |
| 145 | cd ${1} |
| 146 | # expect only one sub-directory, starting with a digit, plus maven-metadata.xml*: |
| 147 | other=`ls -d [!0-9]* 2> /dev/null` |
Stephan Herrmann | 0b6f393 | 2017-01-01 21:17:57 +0100 | [diff] [blame] | 148 | if echo "${other}" | tr "\n" " " | egrep "^maven-metadata.xml maven-metadata.xml.md5 maven-metadata.xml.sha1 \$" |
Stephan Herrmann | ee73d53 | 2016-12-20 23:37:28 +0100 | [diff] [blame] | 149 | then |
Stephan Herrmann | 0a35892 | 2017-01-01 11:19:36 +0100 | [diff] [blame] | 150 | : # clean -> proceed below |
Stephan Herrmann | 9a98101 | 2017-01-01 11:05:54 +0100 | [diff] [blame] | 151 | else |
| 152 | exit 1 # unexpected content found, don't remove |
Stephan Herrmann | ee73d53 | 2016-12-20 23:37:28 +0100 | [diff] [blame] | 153 | fi |
| 154 | # scan all *.pom inside the version sub-directory |
| 155 | r=1 |
| 156 | for pom in `ls [0-9]*/*.pom 2> /dev/null` |
| 157 | do |
| 158 | jar=`echo ${pom} | sed -e "s|\(.*\)\.pom|\1.jar|"` |
| 159 | if [ -f ${jar} ] |
| 160 | then |
| 161 | # jar found, so keep it |
| 162 | exit 1 |
| 163 | fi |
| 164 | # pom without jar found, let's answer true below |
| 165 | r=0 |
| 166 | done |
| 167 | exit $r |
| 168 | } |
| 169 | export -f hasPomButNoJar |
| 170 | |
| 171 | /usr/bin/find org/eclipse/{jdt,pde,platform} -type d \ |
| 172 | -exec /bin/bash -c 'hasPomButNoJar "$@"' bash {} \; \ |
| 173 | -print -exec /bin/rm -rf {} \; -prune >> .logs/removed.txt |
| 174 | # second "bash" is used as $0 in the function |
| 175 | |
| 176 | cd ${WORKSPACE} |
| 177 | |
| 178 | echo "========== Repo reduced: ==========" |
| 179 | /usr/bin/du -sc ${Repo}/* |
| 180 | /usr/bin/du -sc ${Repo}/org/* |
| 181 | /usr/bin/du -sc ${Repo}/org/eclipse/* |
| 182 | echo "===================================" |
| 183 | |
| 184 | #================================================================================ |
| 185 | # (2) Garbage Collector |
| 186 | #================================================================================ |
| 187 | # Removes from the build output of cbiAggregator everything that is not referenced |
| 188 | # from any pom below org/eclipse/{platform,jdt,pde} |
| 189 | # |
| 190 | # Log output: |
| 191 | # .logs/removedGarbage.txt all directories during garbage collection |
| 192 | # .logs/gc.log incoming dependencies of retained artifacts |
| 193 | # .logs/empty-dirs.txt removed empty directories |
| 194 | |
| 195 | echo "==== Garbage Collector ====" |
| 196 | |
| 197 | cd ${Repo} |
| 198 | |
| 199 | #==== function gc_bundle(): ==== |
| 200 | # Test if pom ${1} is referenced in any other pom. |
| 201 | # If not, append the containing directory to the file "toremove.txt" |
| 202 | function gc_bundle { |
| 203 | AID=`echo ${1} | sed -e "s|.*/\(.*\)[_-].*|\1|"` |
| 204 | DIR=`echo ${1} | sed -e "s|\(.*\)/[0-9].*|\1|"` |
| 205 | POM=`basename ${1}` |
| 206 | |
| 207 | ANSWER=`find org/eclipse/{platform,jdt,pde} -name \*.pom \! -name ${POM} \ |
| 208 | -exec /bin/grep -q "<artifactId>${AID}</artifactId>" {} \; -print -quit` |
| 209 | |
| 210 | if [ "$ANSWER" == "" ] |
| 211 | then |
| 212 | echo "Will remove $DIR" |
| 213 | echo $DIR >> toremove.txt |
| 214 | else |
| 215 | echo "$1 is used by $ANSWER" |
| 216 | fi |
| 217 | } |
| 218 | export -f gc_bundle |
| 219 | |
| 220 | #==== run the garbage collector: ==== |
| 221 | # iterate (max 5 times) in case artifacts were used only from garbage: |
| 222 | for iteration in 1 2 3 4 5 |
| 223 | do |
| 224 | echo "== GC iteration ${iteration} ==" |
| 225 | |
| 226 | # look for garbage only outside platform, jdt or pde folders: |
| 227 | find -name platform -prune -o -name jdt -prune -o -name pde -prune -o \ |
| 228 | -name \*.pom -exec /bin/bash -c 'gc_bundle "$@"' bash {} \; \ |
| 229 | > gc-${iteration}.log |
| 230 | # second "bash" is used as $0 in the function |
| 231 | |
| 232 | if [ ! -f toremove.txt ] |
| 233 | then |
| 234 | # no more garbage found |
| 235 | break |
| 236 | fi |
| 237 | cat toremove.txt >> .logs/removedGarbage.txt |
| 238 | for d in `cat toremove.txt`; do /bin/rm -r $d; done |
| 239 | /bin/rm toremove.txt |
| 240 | done |
| 241 | |
| 242 | # merge gc logs: |
| 243 | cat gc-*.log | sort --unique > .logs/gc.log |
| 244 | /bin/rm gc-*.log |
| 245 | |
| 246 | #==== remove all directories that have become empty: ==== |
| 247 | for iteration in 1 2 3 4 5 ; do find -type d -empty -print \ |
| 248 | -exec /bin/rmdir {} \; -prune; done \ |
| 249 | >> .logs/empty-dirs.txt |
| 250 | |
| 251 | echo "========== Repo reduced: ==========" |
| 252 | /usr/bin/du -sc ${Repo}/* |
| 253 | /usr/bin/du -sc ${Repo}/org/* |
| 254 | /usr/bin/du -sc ${Repo}/org/eclipse/* |
| 255 | echo "===================================" |
| 256 | |
| 257 | cd ${WORKSPACE} |
| 258 | |
| 259 | #================================================================================ |
| 260 | # (3) Enrich POMs |
| 261 | #================================================================================ |
| 262 | # Add some required information to the generated poms: |
| 263 | # - dynamic content (retrieved mostly from MANIFEST.MF): |
| 264 | # - name |
| 265 | # - url |
Stephan Herrmann | 9a98101 | 2017-01-01 11:05:54 +0100 | [diff] [blame] | 266 | # - scm connection, tag and url |
| 267 | # - semi dynamic |
| 268 | # - developers (based on static map git-repo-base -> project leads) |
Stephan Herrmann | ee73d53 | 2016-12-20 23:37:28 +0100 | [diff] [blame] | 269 | # - static content |
| 270 | # - license |
| 271 | # - organization |
| 272 | # - issue management |
| 273 | |
| 274 | |
| 275 | echo "==== Enrich POMs ====" |
| 276 | |
| 277 | cd ${Repo} |
| 278 | |
| 279 | echo "platform" |
| 280 | java -jar ${ENRICH_POMS_JAR} `pwd`/org/eclipse/platform &> .logs/enrich-platform.txt |
| 281 | echo "jdt" |
| 282 | java -jar ${ENRICH_POMS_JAR} `pwd`/org/eclipse/jdt &> .logs/enrich-jdt.txt |
| 283 | echo "pde" |
| 284 | java -jar ${ENRICH_POMS_JAR} `pwd`/org/eclipse/pde &> .logs/enrich-pde.txt |
| 285 | |
| 286 | echo "== updated checksums ==" |
| 287 | |
| 288 | function updateCheckSums() { |
| 289 | /usr/bin/md5sum ${1} | cut -d " " -f 1 > ${1}.md5 |
| 290 | /usr/bin/sha1sum ${1} | cut -d " " -f 1 > ${1}.sha1 |
| 291 | } |
| 292 | |
| 293 | for pom in org/eclipse/{platform,jdt,pde}/*/*/*.pom |
| 294 | do |
| 295 | updateCheckSums ${pom} |
| 296 | done |
| 297 | |
Stephan Herrmann | 9a98101 | 2017-01-01 11:05:54 +0100 | [diff] [blame] | 298 | echo "==== Add Javadoc stubs ====" |
Stephan Herrmann | 42f24c9 | 2016-12-31 16:53:10 +0100 | [diff] [blame] | 299 | |
Stephan Herrmann | 07ae09f | 2017-01-01 18:58:28 +0100 | [diff] [blame] | 300 | cd ${Repo} |
| 301 | |
Stephan Herrmann | 42f24c9 | 2016-12-31 16:53:10 +0100 | [diff] [blame] | 302 | # (groupSimpleName, javadocArtifactGA) |
| 303 | function createJavadocs() { |
| 304 | group=${1} |
| 305 | jar="${1}-javadoc.jar" |
| 306 | artifact=${2} |
| 307 | if [ -r ${jar} ] |
| 308 | then |
| 309 | /bin/rm ${jar} |
| 310 | fi |
Stephan Herrmann | 9fc05c9 | 2016-12-31 17:35:26 +0100 | [diff] [blame] | 311 | echo -e "Corresponding javadoc can be found in artifact ${artifact}\n" > README.txt |
Stephan Herrmann | 42f24c9 | 2016-12-31 16:53:10 +0100 | [diff] [blame] | 312 | jar cf ${jar} README.txt |
| 313 | for pom in org/eclipse/${group}/*/*/*.pom |
| 314 | do |
| 315 | javadoc=`echo ${pom} | sed -e "s|\(.*\)\.pom|\1-javadoc.jar|"` |
| 316 | /bin/cp ${jar} ${javadoc} |
| 317 | done |
| 318 | } |
| 319 | |
| 320 | createJavadocs platform org.eclipse.platform:org.eclipse.platform.doc.isv |
| 321 | createJavadocs jdt org.eclipse.jdt:org.eclipse.jdt.doc.isv |
| 322 | createJavadocs pde org.eclipse.pde:org.eclipse.pde.doc.user |
Stephan Herrmann | ee73d53 | 2016-12-20 23:37:28 +0100 | [diff] [blame] | 323 | |
Stephan Herrmann | 07ae09f | 2017-01-01 18:58:28 +0100 | [diff] [blame] | 324 | echo "==== Create missing sources artifacts ====" |
| 325 | |
| 326 | cd ${Repo} |
| 327 | |
| 328 | function buildSourceJar() { |
| 329 | if [ -d assemble ] |
| 330 | then |
| 331 | /bin/rm -r assemble |
| 332 | fi |
| 333 | /bin/mkdir assemble |
Stephan Herrmann | c6c07eb | 2017-01-01 20:50:11 +0100 | [diff] [blame] | 334 | giturl=file://localhost/gitroot/${1} |
Stephan Herrmann | 07ae09f | 2017-01-01 18:58:28 +0100 | [diff] [blame] | 335 | gitpath=${2} |
| 336 | gittag=${3} |
| 337 | group=${4} |
| 338 | artifact=${5} |
| 339 | version=${6} |
| 340 | git archive --remote=${giturl} \ |
| 341 | ${gittag} ${gitpath} \ |
| 342 | | tar xv |
| 343 | /bin/mv ${gitpath}/src/* assemble/ |
| 344 | /bin/mv ${gitpath}/META-INF assemble/ |
| 345 | if [ -d ${gitpath}/OSGI-INF ] |
| 346 | then |
| 347 | /bin/mv ${gitpath}/OSGI-INF assemble/ |
| 348 | fi |
| 349 | /bin/mv ${gitpath}/about.html assemble/ |
Stephan Herrmann | c6c07eb | 2017-01-01 20:50:11 +0100 | [diff] [blame] | 350 | if [ $# > 6 ] |
| 351 | then |
| 352 | shift 6 |
| 353 | for src in "$@" |
| 354 | do |
| 355 | /bin/mv "${gitpath}/${src}" assemble/ |
| 356 | done |
| 357 | fi |
Stephan Herrmann | 07ae09f | 2017-01-01 18:58:28 +0100 | [diff] [blame] | 358 | /bin/rm -rf ${gitpath} |
| 359 | cd assemble |
| 360 | jar cf ../${group}/${artifact}/${version}/${artifact}-${version}-sources.jar * |
| 361 | cd - |
| 362 | } |
| 363 | |
Stephan Herrmann | 4e843b0 | 2017-01-26 21:18:42 +0100 | [diff] [blame^] | 364 | while read line |
| 365 | do |
| 366 | buildSourceJar $line |
| 367 | done < `dirname $0`/sourceBundles.txt |
Stephan Herrmann | 07ae09f | 2017-01-01 18:58:28 +0100 | [diff] [blame] | 368 | |
Stephan Herrmann | 4e843b0 | 2017-01-26 21:18:42 +0100 | [diff] [blame^] | 369 | # special hack for missing source bundle with several source folders with blanks: |
Stephan Herrmann | c6c07eb | 2017-01-01 20:50:11 +0100 | [diff] [blame] | 370 | buildSourceJar platform/eclipse.platform.swt.git \ |
| 371 | bundles/org.eclipse.swt.tools \ |
| 372 | M20161109-0400 \ |
| 373 | org/eclipse/platform org.eclipse.swt.tools 3.105.2 \ |
| 374 | "JNI Generation" \ |
| 375 | "Mac Generation" \ |
| 376 | "Icon Exe" \ |
| 377 | "NativeStats" \ |
| 378 | "Mozilla Generation" \ |
| 379 | "JavadocBasher" |
| 380 | |
Stephan Herrmann | ee73d53 | 2016-12-20 23:37:28 +0100 | [diff] [blame] | 381 | echo "========== Repo completed =========" |
| 382 | |
| 383 | cd ${WORKSPACE} |