Skip to main content
aboutsummaryrefslogtreecommitdiffstats
blob: 6644aaaa892de4150644664f752e580d27929950 (plain) (blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
#!/usr/bin/env bash


if [ $# -ne 1 ]; then
  echo USAGE: $0 env_file
  exit 1
fi

INITIAL_ENV_FILE=$1


if [ ! -r "$INITIAL_ENV_FILE" ]; then
  echo "$INITIAL_ENV_FILE" cannot be read
  echo USAGE: $0 env_file
  exit 1
fi


export SCRIPT_PATH="${BUILD_ROOT}/production"


source "${SCRIPT_PATH}/build-functions.shsource"

source "${INITIAL_ENV_FILE}"

source "${SCRIPT_PATH}/bashUtilities.shsource"
source "${SCRIPT_PATH}/bootstrapVariables.shsource"


assertNotEmpty gitCache
assertNotEmpty aggDir
assertNotEmpty BUILD_ID
assertNotEmpty buildDirectory



# remember, local "test builds" that use this script must change
# or override 'GIT_PUSH' to simply echo, not actually push. Only
# e4Build should 'push' these tags.
#GIT_PUSH='echo git push'
if [[ "${testbuildonly}" == "true" ]]
then
  GIT_PUSH='echo no git push since testbuildonly'
fi

if [[ "${BUILD_TYPE}" == "N" ]]
then
  GIT_PUSH='echo no git push done since Nightly'
fi
if [[ "${BUILD_TYPE}" =~ [XY] ]]
then
  GIT_PUSH='echo no git push done since Experimental build'
fi
GIT_PUSH=${GIT_PUSH:-'git push'}


cd $BUILD_ROOT

buildrc=0

# derived values

# correct values for cbi-jdt-repo.url and cbi-jdt-version are
# codified in the parent pom. These variables give an easy way
# to test "experimental versions" in production-like build.

if [[ -n $CBI_JDT_REPO_URL ]]
then
  export CBI_JDT_REPO_URL_ARG="-Dcbi-jdt-repo.url=$CBI_JDT_REPO_URL"
fi
if [[ -n $CBI_JDT_VERSION ]]
then
  export CBI_JDT_VERSION_ARG="-Dcbi-jdt-version=$CBI_JDT_VERSION"
fi

assertNotEmpty buildDirectory
echo "buildDirectory: >${buildDirectory}<"

export logsDirectory="${buildDirectory}/buildlogs"
mkdir -p "${logsDirectory}"
checkForErrorExit $? "Could not create buildlogs directory: ${logsDirectory}"

LOG=$buildDirectory/buildlogs/buildOutput.txt
#exec >>$LOG 2>&1

BUILD_PRETTY_DATE=$( date --date='@'$RAWDATE )
TIMESTAMP=$( date +%Y%m%d-%H%M --date='@'$RAWDATE )

# TRACE_OUTPUT is not normally used. But, it comes in handy for debugging
# when output from some functions can not be written to stdout or stderr
# (due to the nature of the function ... it's "output" being its returned value).
# When needed for local debugging, usually more convenient to provide
# a value relative to PWD in startup scripts.
export TRACE_OUTPUT=${TRACE_OUTPUT:-$buildDirectory/buildlogs/trace_output.txt}
echo $BUILD_PRETTY_DATE > ${TRACE_OUTPUT}

assertNotEmpty buildDirectory

# These files have variable/value pairs for this build, suitable for use in
# shell scripts, PHP files, or as Ant (or Java) properties
export BUILD_ENV_FILE=${buildDirectory}/buildproperties.shsource
export BUILD_ENV_FILE_PHP=${buildDirectory}/buildproperties.php
export BUILD_ENV_FILE_PROP=${buildDirectory}/buildproperties.properties

export LOCAL_REPO="${BUILD_ROOT}"/localMavenRepo

# In production builds, we normally specify CLEAN_LOCAL,
# and remove any existing LOCAL_REPO, and re-fetch.
# But CLEAN_LOCAL can be overridden for remote builds, quick test builds,
# etc.
export CLEAN_LOCAL=${CLEAN_LOCAL:-true}
# We "remove" by moving to backup, in case there's ever any reason to
# compare "what's changed".
if [[ -d ${LOCAL_REPO} && "${CLEAN_LOCAL}" == "true" ]]
then
  # remove existing backup, if it exists
  rm -fr ${LOCAL_REPO}.bak 2>/dev/null
  mv ${LOCAL_REPO} ${LOCAL_REPO}.bak
fi
export STREAMS_PATH="${aggDir}/streams"

BUILD_TYPE_NAME="Integration"
if [ "$BUILD_TYPE" = M ]; then
  BUILD_TYPE_NAME="Maintenance"
elif [ "$BUILD_TYPE" = N ]; then
  BUILD_TYPE_NAME="Nightly (HEAD)"
elif [ "$BUILD_TYPE" = X ]; then
  BUILD_TYPE_NAME="Experimental Branch"
elif [ "$BUILD_TYPE" = Y ]; then
  BUILD_TYPE_NAME="Experimental Branch"
elif [ "$BUILD_TYPE" = P ]; then
  BUILD_TYPE_NAME="Patch"
elif [ "$BUILD_TYPE" = S ]; then
  BUILD_TYPE_NAME="Stable (Milestone)"
fi

# we need this value for later, post build processing, so want it saved to variables file.
# We can either compute here,
# or just make sure it matches that is in parent pom for various streams (e.g. 4.3 vs 4.4)
# and build types, (e.g. M vs. I). For N builds, we still use "I".
# 5/31013. We no longer do the post build comparator, but leaving it, just
# in case we want an occasionally double check ... but, should never be routine.
#comparatorRepository=http://download.eclipse.org/eclipse/updates/4.4-I-builds
comparatorRepository=NOT_CURRENTLY_USED

GET_AGGREGATOR_BUILD_LOG="${logsDirectory}/mb010_get-aggregator_output.txt"
TAG_BUILD_INPUT_LOG="${logsDirectory}/mb030_tag_build_input_output.txt"
POM_VERSION_UPDATE_BUILD_LOG="${logsDirectory}/mb050_pom-version-updater_output.txt"
RUN_MAVEN_BUILD_LOG="${logsDirectory}/mb060_run-maven-build_output.txt"
GATHER_PARTS_BUILD_LOG="${logsDirectory}/mb070_gather-parts_output.txt"

# These "build_dir" variables are needed for checksum URLs, in especially
# after promotion to milestone or release. (see bug 435671)
export BUILD_DIR_SEG=$BUILD_ID
export EQ_BUILD_DIR_SEG=$BUILD_ID

# These variables, from original env file, are re-written to BUILD_ENV_FILE,
# with values for this build (some of them computed) partially for documentation, and
# partially so this build can be re-ran or re-started using it, instead of
# original env file, which would compute different values (in some cases).
# The function also writes into appropriate PHP files and Properties files.
# Init once, here at beginning, but don't close until much later since other functions
# may write variables at various points
fn-write-property-init
fn-write-property PATH
fn-write-property INITIAL_ENV_FILE
fn-write-property BUILD_ROOT
fn-write-property BRANCH
fn-write-property STREAM
fn-write-property aggDir
fn-write-property BUILD_ID
fn-write-property BUILD_TYPE
fn-write-property TIMESTAMP
fn-write-property TMP_DIR
fn-write-property JAVA_HOME
fn-write-property MAVEN_OPTS
fn-write-property MAVEN_PATH
fn-write-property AGGREGATOR_REPO
fn-write-property BASEBUILDER_TAG
fn-write-property B_GIT_EMAIL
fn-write-property B_GIT_NAME
fn-write-property COMMITTER_ID
fn-write-property MVN_DEBUG
fn-write-property MVN_QUIET
fn-write-property SIGNING
fn-write-property REPO_AND_ACCESS
fn-write-property MAVEN_BREE
fn-write-property GIT_PUSH
fn-write-property LOCAL_REPO
fn-write-property SCRIPT_PATH
fn-write-property STREAMS_PATH
fn-write-property CBI_JDT_REPO_URL
fn-write-property CBI_JDT_REPO_URL_ARG
fn-write-property CBI_JDT_VERSION
fn-write-property CBI_JDT_VERSION_ARG
fn-write-property PATCH_BUILD
fn-write-property ALT_POM_FILE
fn-write-property JAVA_DOC_TOOL
# any value of interest/usefulness can be added to BUILD_ENV_FILE
if [[ "${testbuildonly}" == "true" ]]
then
  fn-write-property testbuildonly
fi
fn-write-property buildDirectory
fn-write-property BUILD_ENV_FILE
fn-write-property BUILD_ENV_FILE_PHP
fn-write-property BUILD_ENV_FILE_PROP
fn-write-property BUILD_ID
fn-write-property BUILD_DIR_SEG
fn-write-property EQ_BUILD_DIR_SEG
fn-write-property BUILD_PRETTY_DATE
fn-write-property BUILD_TYPE_NAME
fn-write-property TRACE_OUTPUT
fn-write-property comparatorRepository
fn-write-property logsDirectory
fn-write-property BUILD_HOME


$SCRIPT_PATH/get-aggregator.sh $BUILD_ENV_FILE 2>&1 | tee ${GET_AGGREGATOR_BUILD_LOG}
# if file exists, then get-aggregator failed
if [[ -f "${buildDirectory}/buildFailed-get-aggregator" ]]
then
  buildrc=1
  # Git sometimes fails (returns non-zero error codes) for "warnings".
  # In most cases, but not all, these would be considered errors.
  /bin/grep  "^warning: \|\[ERROR\]"  "${GET_AGGREGATOR_BUILD_LOG}" >> "${buildDirectory}/buildFailed-get-aggregator"
  BUILD_FAILED="${GET_AGGREGATOR_BUILD_LOG}"
  fn-write-property BUILD_FAILED
else
  # if get-aggregator failed, there is no reason to try and update input or run build
  $SCRIPT_PATH/update-build-input.sh $BUILD_ENV_FILE 2>&1 | tee $logsDirectory/mb020_update-build-input_output.txt
  checkForErrorExit $? "Error occurred while updating build input"


  # We always make tag commits, if build successful or not, but don't push
  # back to origin if doing N builds or test builds.
  pushd "$aggDir"
  git commit -m "Build input for build $BUILD_ID"
  echo "RC from commit with $BUILD_ID: $?"
  # exits with 1 here ... with warning, if commit already exists?
  #checkForErrorExit $? "Error occurred during commit of build_id"

  # GIT_PUSH just echos, for N builds and test builds
  $GIT_PUSH origin HEAD
  #checkForErrorExit $? "Error occurred during push of build_id commit"
  popd

  $SCRIPT_PATH/tag-build-input.sh $BUILD_ENV_FILE 2>&1 | tee $TAG_BUILD_INPUT_LOG
  checkForErrorExit $? "Error occurred during tag of build input"

  # At this point, everything should be checked out, updated, and tagged
  # (tagged unless N build or test build)
  # So is a good point to capture listing of build input to directory.txt file.
  # TODO: hard to find good/easy git commands that work for any repo,
  # to query actual branches/commits/tags on remote, in a reliable way?
  pushd "$aggDir"
  # = = = = directtory.txt section
  echo "# Build ${BUILD_ID}, ${BUILD_PRETTY_DATE}" > ${buildDirectory}/directory.txt

  echo "# " >> ${buildDirectory}/directory.txt
  echo "# This is simply a listing of repositories.txt. Remember that for N-builds, "  >> ${buildDirectory}/directory.txt
  echo "# 'master' is always used, and N-builds are not tagged."  >> ${buildDirectory}/directory.txt
  echo "# I and M builds are tagged with buildId: ${BUILD_ID} "  >> ${buildDirectory}/directory.txt
  echo "# (when repository is a branch, which it typcally is)."  >> ${buildDirectory}/directory.txt
  echo "# " >> ${buildDirectory}/directory.txt

  if [[ $BUILD_TYPE =~ [IMXYP] ]]
  then
    AGGRCOMMIT=$( git rev-parse HEAD )
    echo "eclipse.platform.releng.aggregator TAGGED: ${BUILD_ID}"  >> ${buildDirectory}/directory.txt
    echo "       http://git.eclipse.org/c/platform/eclipse.platform.releng.aggregator.git/commit/?id=${AGGRCOMMIT}"  >> ${buildDirectory}/directory.txt
  fi

  echo "# " >> ${buildDirectory}/directory.txt
  echo "# .../streams/repositories${PATCH_BUILD}.txt" >> ${buildDirectory}/directory.txt
  echo "# " >> ${buildDirectory}/directory.txt
  cat $STREAMS_PATH/repositories${PATCH_BUILD}.txt >> ${buildDirectory}/directory.txt
  echo "# " >> ${buildDirectory}/directory.txt


  # = = = = relengirectory.txt section
  echo "# " >> ${logsDirectory}/relengdirectory.txt
  echo "# Build Input: " >> ${logsDirectory}/relengdirectory.txt
  $SCRIPT_PATH/git-doclog >> ${logsDirectory}/relengdirectory.txt
  git submodule foreach --quiet $SCRIPT_PATH/git-doclog >> ${logsDirectory}/relengdirectory.txt
  echo "# " >> ${logsDirectory}/relengdirectory.txt
  popd


  $SCRIPT_PATH/pom-version-updater.sh $BUILD_ENV_FILE 2>&1 | tee ${POM_VERSION_UPDATE_BUILD_LOG}
  # if file exists, pom update failed
  if [[ -f "${buildDirectory}/buildFailed-pom-version-updater" ]]
  then
    buildrc=1
    /bin/grep "\[ERROR\]" "${POM_VERSION_UPDATE_BUILD_LOG}" >> "${buildDirectory}/buildFailed-pom-version-updater"
    echo "BUILD FAILED. See ${POM_VERSION_UPDATE_BUILD_LOG}."
    BUILD_FAILED=${POM_VERSION_UPDATE_BUILD_LOG}
    fn-write-property BUILD_FAILED
  else
    # if updater failed, something fairly large is wrong, so no need to compile
    $SCRIPT_PATH/run-maven-build.sh $BUILD_ENV_FILE 2>&1 | tee ${RUN_MAVEN_BUILD_LOG}
    # if file exists, then run maven build failed.
    if [[ -f "${buildDirectory}/buildFailed-run-maven-build" ]]
    then
      buildrc=1
      /bin/grep "\[ERROR\]" "${RUN_MAVEN_BUILD_LOG}" >> "${buildDirectory}/buildFailed-run-maven-build"
      BUILD_FAILED=${RUN_MAVEN_BUILD_LOG}
      fn-write-property BUILD_FAILED
      # TODO: eventually put in more logic to "track" the failure, so
      # proper actions and emails can be sent. For example, we'd still want to
      # publish what we have, but not start the tests.
      echo "BUILD FAILED. See ${RUN_MAVEN_BUILD_LOG}."
    else
      # if build run maven build failed, no need to gather parts
      $SCRIPT_PATH/gather-parts.sh $BUILD_ENV_FILE 2>&1 | tee ${GATHER_PARTS_BUILD_LOG}
      if [[ -f "${buildDirectory}/buildFailed-gather-parts" ]]
      then
        buildrc=1
        /bin/grep -i "ERROR" "${GATHER_PARTS_BUILD_LOG}" >> "${buildDirectory}/buildFailed-gather-parts"
        BUILD_FAILED=${GATHER_PARTS_BUILD_LOG}
        fn-write-property BUILD_FAILED
        echo "BUILD FAILED. See ${GATHER_PARTS_BUILD_LOG}."
      fi
    fi
  fi
fi

$SCRIPT_PATH/publish-eclipse.sh $BUILD_ENV_FILE >$logsDirectory/mb080_publish-eclipse_output.txt
checkForErrorExit $? "Error occurred during publish-eclipse"


# We don't publish repo if there was a build failure, it likely doesn't exist.
if [[ -z "${BUILD_FAILED}" ]]
then
  $SCRIPT_PATH/publish-repo.sh $BUILD_ENV_FILE >$logsDirectory/mb083_publish-repo_output.txt
  checkForErrorExit $? "Error occurred during publish-repo"
else
  echo "No repo published, since BUILD_FAILED"
fi

#For now, only "publish equinox and promote" if N, I or M build, skip if P, X, or Y

# TODO: probably never need to promote equinox, for patch build?
# TODO: Unclear how/when to send mailing list notification for patch builds.

if [[ $BUILD_TYPE =~  [NIM] ]]
then

  # We don't promote equinox if there was a build failure, and we should not even try to
  # create the site locally, because it depends heavily on having a valid repository to
  # work from.
  if [[ -z "${BUILD_FAILED}" ]]
  then
    $SCRIPT_PATH/publish-equinox.sh $BUILD_ENV_FILE >$logsDirectory/mb085_publish-equinox_output.txt
    checkForErrorExit $? "Error occurred during publish-equinox"
  fi
fi

# if all ended well, put "promotion scripts" in known locations
$SCRIPT_PATH/promote-build.sh $BUILD_ENV_FILE 2>&1 | tee $logsDirectory/mb090_promote-build_output.txt
checkForErrorExit $? "Error occurred during promote-build"

# check for dirt in working tree. Note. we want near very end, since even things 
# like "publishing" in theory could leave dirt behind. 
$SCRIPT_PATH/dirtReport.sh $BUILD_ENV_FILE >$logsDirectory/dirtReport.txt
checkForErrorExit $? "Error occurred during dirt report"

fn-write-property-close

# dump ALL environment variables in case its helpful in documenting or
# debugging build results or differences between runs, especially on different machines
env 1>$logsDirectory/mb100_all-env-variables_output.txt

echo "Exiting build with RC code of $buildrc"
exit $buildrc

Back to the top