Bug 404593 - misc. minor improvements in production scripts (cleanup,

standard formatting, added error check in functions)

17 files changed, 164 insertions, 127 deletions

diff --git a/production/build-functions.sh b/production/build-functions.shsource
index 3e1daab2d..8f168be4e 100755
--- a/production/build-functions.sh
+++ b/production/build-functions.shsource
@@ -1,6 +1,26 @@
 #!/usr/bin/env bash
 # this is not really to be executed, but sourced where needed
 
+# A general purpose utility to check number of arguments match
+# what was expected. Its purpose is to help bullet-proof these scripts
+# when future changes made. 
+# Example: (such as in first line of a function)
+#   checkNArgs $# 3
+checkNArgs ()
+{
+    actual=$1
+    expected=$2
+    #echo "     DEBUG checkNArgs funcName[1]: ${FUNCNAME[1]}" >> ${TRACE_OUTPUT}
+    #echo "     DEBUG checkNArgs bashSource[2]:   ${BASH_SOURCE[2]}" >> ${TRACE_OUTPUT}
+    #echo "     DEBUG checkNArgs bashLineNo[1]:   ${BASH_LINENO[1]}" >> ${TRACE_OUTPUT}
+    if [[ $actual != $expected ]]
+    then
+        echo "PROGRAM ERROR: expected $expected arguments but was passed $actual."
+        echo "               Called from ${FUNCNAME[1]}, called from line number ${BASH_LINENO[1]} in ${BASH_SOURCE[2]}."
+        exit 1
+    fi
+}
+
 # general purpose utility for "hard exit" if return code not zero.
 # especially useful to call/check after basic things that should normally
 # easily succeeed.
@@ -159,8 +179,10 @@ fn-git-reset-submodules ()
 
 # USAGE: fn-build-id BUILD_TYPE
 #   BUILD_TYPE: I, M, N
+# TODO: depends on RAWDATE being exported/global
 fn-build-id () 
 {
+    checkNArgs $# 1
     BUILD_TYPE="$1"; shift
     TIMESTAMP=$( date +%Y%m%d-%H%M --date='@'$RAWDATE )
     echo ${BUILD_TYPE}${TIMESTAMP}
@@ -169,8 +191,10 @@ fn-build-id ()
 # USAGE: fn-local-repo URL [TO_REPLACE]
 #   URL: git://git.eclipse.org/gitroot/platform/eclipse.platform.releng.aggregator.git
 #   TO_REPLACE: git://git.eclipse.org
+# TODO: since we compute here, do we really need to define in build_eclipse_org.shsource?
 fn-local-repo () 
 {
+    checkNArgs $# 2
     FORCE_LOCAL_REPO=false
     URL="$1"; shift
     if [ build = $(hostname) -o $FORCE_LOCAL_REPO = true ]; then
@@ -188,6 +212,7 @@ fn-local-repo ()
 #   TO_REPLACE: git://git.eclipse.org
 fn-local-repos () 
 {
+    checkNArgs $# 1
     TO_REPLACE='git://git.eclipse.org'
     if [ $# -gt 0 ]; then
         TO_REPLACE="$1"; shift
@@ -201,6 +226,7 @@ fn-local-repos ()
 #   BRANCH: R4_2_maintenance
 fn-git-clone-aggregator () 
 {
+    checkNArgs $# 3
     GIT_CACHE="$1"; shift
     URL="$1"; shift
     BRANCH="$1"; shift
@@ -220,6 +246,7 @@ fn-git-clone-aggregator ()
 #   BRANCH: R4_2_maintenance
 fn-git-clean-aggregator () 
 {
+    checkNArgs $# 2
     AGGREGATOR_DIR="$1"; shift
     BRANCH="$1"; shift
     pushd "$AGGREGATOR_DIR"
@@ -254,6 +281,7 @@ fn-git-clean-aggregator ()
 #   BRANCH: R4_2_maintenance
 fn-git-cache () 
 {
+    checkNArgs $# 2
     ROOT="$1"; shift
     BRANCH="$1"; shift
     echo $ROOT/$BRANCH/gitCache
@@ -264,6 +292,7 @@ fn-git-cache ()
 #   URL: file:///gitroot/platform/eclipse.platform.releng.aggregator.git
 fn-git-dir () 
 {
+    checkNArgs $# 2
     GIT_CACHE="$1"; shift
     URL="$1"; shift
     echo $GIT_CACHE/$( basename "$URL" .git )
@@ -275,11 +304,7 @@ fn-git-dir ()
 #   STREAM: 4.3.0
 fn-build-dir () 
 {
-    if [[ $# != 3 ]]
-    then
-        exit "This function, fn-build-dir, requires 3 arguments, but $# provided."
-        exit 1
-    fi
+    checkNArgs $# 3
     ROOT="$1"; shift
     BUILD_ID="$1"; shift
     STREAM="$1"; shift
@@ -298,11 +323,7 @@ fn-build-dir ()
 #   STREAM: 4.2.2
 fn-basebuilder-dir () 
 {
-    if [[ $# != 3 ]] 
-    then 
-        echo "PROGRAM ERROR: this function, fn-basebuilder-dir, requires 3 arguments"
-        exit 1
-    fi
+    checkNArgs $# 3
     ROOT="$1"; shift
     BUILD_ID="$1"; shift
     STREAM="$1"; shift
@@ -318,8 +339,10 @@ fn-basebuilder-dir ()
 #   VERBOSE: true
 #   SIGNING: true
 #   UPDATE_BRANDING: true
+# TODO: depends on BUILD_TYPE being exported/global
 fn-maven-build-aggregator () 
 {
+    checkNArgs $# 8
     BUILD_ID="$1"; shift
     REPO_DIR="$1"; shift
     LOCAL_REPO="$1"; shift
@@ -383,6 +406,7 @@ fn-maven-build-aggregator ()
 #   REPOSITORIES_TXT: /shared/eclipse/builds/streams/repositories.txt
 fn-submodule-checkout () 
 {
+    checkNArgs $# 4
     BUILD_ID="$1"; shift
     REPO_DIR="$1"; shift
     SCRIPT="$1"; shift
@@ -411,6 +435,7 @@ fn-submodule-checkout ()
 #   REPO_DIR: /shared/eclipse/builds/R4_2_maintenance/gitCache/eclipse.platform.releng.aggregator
 fn-add-submodule-updates () 
 {
+    checkNArgs $# 1
     REPO_DIR="$1"; shift
     pushd "$REPO_DIR"
     adds=$( git submodule | grep "^+" | cut -f2 -d" " )
@@ -429,6 +454,7 @@ fn-add-submodule-updates ()
 #   REPOSITORIES_TXT: /shared/eclipse/builds/streams/repositories.txt
 fn-tag-build-inputs () 
 {
+    checkNArgs $# 3
     BUILD_ID="$1"; shift
     REPO_DIR="$1"; shift
     REPOSITORIES_TXT="$1"; shift
@@ -442,8 +468,10 @@ fn-tag-build-inputs ()
 # USAGE: fn-pom-version-updater REPO_DIR LOCAL_REPO DEBUG QUIET
 #   REPO_DIR: /shared/eclipse/builds/R4_2_maintenance/gitCache/eclipse.platform.releng.aggregator
 #   LOCAL_REPO: /shared/eclipse/builds/R4_2_maintenance/localMavenRepo
+# TODO: depends on TMP_DIR being exported/global
 fn-pom-version-updater () 
 {
+    checkNArgs $# 4
     REPO_DIR="$1"; shift
     LOCAL_REPO="$1"; shift
     DEBUG=$1; shift
@@ -498,6 +526,7 @@ fn-pom-version-updater ()
 #   SIGNING: true
 fn-pom-version-update-with-commit () 
 {
+    checkNArgs $# 3
     BUILD_ID="$1"; shift
     REPO_DIR="$1"; shift
     LOCAL_REPO="$1"; shift
@@ -542,6 +571,7 @@ fn-pom-version-update-with-commit ()
 #   BUILD_DIR: /shared/eclipse/builds/R4_2_maintenance/dirs/M20121120-1747
 fn-gather-repo () 
 {
+    checkNArgs $# 3
     BUILD_ID="$1"; shift
     REPO_DIR="$1"; shift
     BUILD_DIR="$1"; shift
@@ -564,6 +594,7 @@ fn-gather-repo ()
 #   BUILD_DIR: /shared/eclipse/builds/R4_2_maintenance/dirs/M20121120-1747
 fn-gather-sdk () 
 {
+    checkNArgs $# 3
     BUILD_ID="$1"; shift
     REPO_DIR="$1"; shift
     BUILD_DIR="$1"; shift
@@ -600,6 +631,7 @@ fn-gather-sdk ()
 #   BUILD_DIR: /shared/eclipse/builds/R4_2_maintenance/dirs/M20121120-1747
 fn-gather-platform () 
 {
+    checkNArgs $# 3
     BUILD_ID="$1"; shift
     REPO_DIR="$1"; shift
     BUILD_DIR="$1"; shift
@@ -634,6 +666,7 @@ fn-gather-platform ()
 #   BUILD_DIR: /shared/eclipse/builds/R4_2_maintenance/dirs/M20121120-1747
 fn-gather-swt-zips () 
 {
+    checkNArgs $# 3
     BUILD_ID="$1"; shift
     REPO_DIR="$1"; shift
     BUILD_DIR="$1"; shift
@@ -656,6 +689,7 @@ fn-gather-swt-zips ()
 #   BUILD_DIR: /shared/eclipse/builds/R4_2_maintenance/dirs/M20121120-1747
 fn-gather-test-zips () 
 {
+    checkNArgs $# 3
     BUILD_ID="$1"; shift
     REPO_DIR="$1"; shift
     BUILD_DIR="$1"; shift
@@ -686,6 +720,7 @@ fn-gather-test-zips ()
 #   BUILD_DIR: /shared/eclipse/builds/R4_2_maintenance/dirs/M20121120-1747
 fn-gather-ecj-jars () 
 {
+    checkNArgs $# 3
     BUILD_ID="$1"; shift
     REPO_DIR="$1"; shift
     BUILD_DIR="$1"; shift
@@ -712,6 +747,7 @@ fn-gather-ecj-jars ()
 #   BASEBUILDER_LAUNCHER: /shared/eclipse/builds/R4_2_maintenance/org.eclipse.releng.basebuilder_R3_7/plugins/org.eclipse.equinox.launcher_1.2.0.v20110502.jar
 fn-slice-repos () 
 {
+    checkNArgs $# 4
     BUILD_ID="$1"; shift
     REPO_DIR="$1"; shift
     BUILD_DIR="$1"; shift
@@ -744,6 +780,7 @@ fn-slice-repos ()
 #   BUILD_DIR: /shared/eclipse/builds/R4_2_maintenance/dirs/M20121120-1747
 fn-gather-repo-zips () 
 {
+    checkNArgs $# 3
     BUILD_ID="$1"; shift
     REPO_DIR="$1"; shift
     BUILD_DIR="$1"; shift
@@ -767,6 +804,7 @@ fn-gather-repo-zips ()
 #   BUILD_DIR: /shared/eclipse/builds/R4_2_maintenance/dirs/M20121120-1747
 fn-gather-compile-logs () 
 {
+    checkNArgs $# 3
     BUILD_ID="$1"; shift
     REPO_DIR="$1"; shift
     BUILD_DIR="$1"; shift
@@ -802,34 +840,18 @@ fn-gather-compile-logs ()
 #   BUILD_PRETTY_DATE: Thu Nov 20 17:47:35 EST 2012
 fn-gather-main-index () 
 {
+    checkNArgs $# 6
     BUILD_ID="$1"; shift
     REPO_DIR="$1"; shift
     BUILD_DIR="$1"; shift
     STREAM="$1"; shift
     BUILD_TYPE="$1"; shift
-    #BUILD_TYPE_NAME=Integration
-    #if [ "$BUILD_TYPE" = M ]; then
-    #    BUILD_TYPE_NAME=Maintenance
-    #fi
     BUILD_PRETTY_DATE="$1"; shift
     pushd "$REPO_DIR"/eclipse.platform.releng.tychoeclipsebuilder/eclipse/templateFiles
 
     # Simplified by creating PHP variables in buildproperties.php
     cp "index.php.template" "$BUILD_DIR"/index.php
 
-    #    # fail fast if not set up correctly
-    #rc=$(fn-check-dir-exists TMP_DIR)
-    #checkForErrorExit "$rc" "$rc"
-
-    #T1=${TMP_DIR}/t1_$$
-    #T2=${TMP_DIR}/t2_$$
-    #sed "s/@eclipseStream@/$STREAM/g" index.php.template >$T1
-    #sed "s/@type@/$BUILD_TYPE_NAME/g" $T1 >$T2
-    #sed "s/@build@/$BUILD_ID/g" $T2 >$T1
-    #sed "s/@date@/$BUILD_PRETTY_DATE/g" $T1 >$T2
-    #sed "s/@buildlabel@/$BUILD_ID/g" $T2 >$T1
-    #cp $T1 "$BUILD_DIR"/index.php
-    #rm $T1 $T2
     popd
 }
 
@@ -840,6 +862,7 @@ fn-gather-main-index ()
 #   BASEBUILDER_LAUNCHER: /shared/eclipse/builds/R4_2_maintenance/org.eclipse.releng.basebuilder_R3_7/plugins/org.eclipse.equinox.launcher_1.2.0.v20110502.jar
 fn-parse-compile-logs () 
 {
+    checkNArgs $# 4
     BUILD_ID="$1"; shift
     ANT_SCRIPT="$1"; shift
     BUILD_DIR="$1"; shift
@@ -865,6 +888,7 @@ fn-parse-compile-logs ()
 #   BASEBUILDER_LAUNCHER: /shared/eclipse/builds/R4_2_maintenance/org.eclipse.releng.basebuilder_R3_7/plugins/org.eclipse.equinox.launcher_1.2.0.v20110502.jar
 fn-publish-eclipse () 
 {
+    checkNArgs $# 6
     BUILD_TYPE="$1"; shift
     BUILD_STREAM="$1"; shift
     BUILD_ID="$1"; shift
@@ -897,10 +921,14 @@ fn-publish-eclipse ()
 #   BASEBUILDER_TAG: R3_7
 fn-checkout-basebuilder () 
 {
+    checkNArgs $# 2
     BUILDER_DIR="$1"; shift
     BASEBUILDER_TAG="$1"; shift
     if [ -e "$BUILDER_DIR" ]; then
-        return
+        return 0
+    else 
+        echo "Basebuilder directory already existed, so not re-fetched."
+        echo "   Directory found at $BUILDER_DIR."
     fi
     pushd $( dirname "$BUILDER_DIR" )
     wget --no-verbose -O basebuilder-${BASEBUILDER_TAG}.zip http://git.eclipse.org/c/platform/eclipse.platform.releng.basebuilder.git/snapshot/eclipse.platform.releng.basebuilder-${BASEBUILDER_TAG}.zip 2>&1
@@ -914,6 +942,7 @@ fn-checkout-basebuilder ()
 #   BUILDER_DIR: /shared/eclipse/builds/R4_2_maintenance/org.eclipse.releng.basebuilder_R3_7
 fn-basebuilder-launcher () 
 {
+    checkNArgs $# 1
     BUILDER_DIR="$1"; shift
     find "$BUILDER_DIR" -name "org.eclipse.equinox.launcher_*.jar" | tail -1
 }
@@ -924,6 +953,7 @@ fn-basebuilder-launcher ()
 #   BUILD_DIR: /shared/eclipse/builds/R4_2_maintenance/dirs/M20121120-1747
 fn-pom-version-report () 
 {
+    checkNArgs $# 3
     BUILD_ID="$1"; shift
     REPO_DIR="$1"; shift
     BUILD_DIR="$1"; shift
@@ -964,6 +994,7 @@ fn-pom-version-report ()
 #        checkForErrorExit "$rc" "$rc"
 fn-check-dir-exists () 
 {
+    checkNArgs $# 1
     DIR_VAR_NAME=$1
     if [[ -z "${!DIR_VAR_NAME}" ]]
     then
@@ -1000,6 +1031,7 @@ fn-check-dir-exists ()
 
 fn-write-property () 
 {
+    checkNArgs $# 1
     VAR_NAME=$1
     if [[ -z "${VAR_NAME}" ]]
     then
@@ -1020,7 +1052,7 @@ fn-write-property ()
 # Must be called (exactly) once before writing properties. 
 fn-write-property-init () 
 {
-
+    checkNArgs $# 0
     # nothing really required for bash shsource, but we'll put in some niceties
     echo "#!/usr/bin/env bash" > $BUILD_ENV_FILE
     echo "# properties written for $BUILD_ID" >> $BUILD_ENV_FILE
@@ -1036,7 +1068,7 @@ fn-write-property-init ()
 # Must be called (exactly) once when completely finished writing properties. 
 fn-write-property-close () 
 {
-
+    checkNArgs $# 0
     # nothing really required for bash shsource, but we'll put in some niceties
     echo "# finished properties for $BUILD_ID" >> $BUILD_ENV_FILE
     # PHP, suitable for direct "include": needs to start and end with <?php ... ?>
diff --git a/production/build_eclipse_org.shsource b/production/build_eclipse_org.shsource
index 9a5b5320f..db5eb935a 100644
--- a/production/build_eclipse_org.shsource
+++ b/production/build_eclipse_org.shsource
@@ -66,8 +66,8 @@ GIT_PUSH=${GIT_PUSH:-'git push'}
 # local, non build.eclipse.org builds (or, test builds) may need to override, and turn off signing.
 # otherwwise, we always sign if I or M build
 if [[ $BUILD_TYPE =~ [IM] ]]
- then
-  SIGNING=${SIGNING:-true}
- else
-  SIGNING=${SIGNING:-false}
+then
+    SIGNING=${SIGNING:-true}
+else
+    SIGNING=${SIGNING:-false}
 fi
diff --git a/production/gather-parts.sh b/production/gather-parts.sh
index 454ddf49c..0786db663 100644
--- a/production/gather-parts.sh
+++ b/production/gather-parts.sh
@@ -2,19 +2,19 @@
 #
 
 if [ $# -ne 1 ]; then
-	echo USAGE: $0 env_file
-	exit 1
+    echo USAGE: $0 env_file
+    exit 1
 fi
 
 if [ ! -r "$1" ]; then
-	echo "$1" cannot be read
-	echo USAGE: $0 env_file
-	exit 1
+    echo "$1" cannot be read
+    echo USAGE: $0 env_file
+    exit 1
 fi
 
 SCRIPT_PATH=${SCRIPT_PATH:-$(pwd)}
 
-source $SCRIPT_PATH/build-functions.sh
+source $SCRIPT_PATH/build-functions.shsource
 
 source "$1"
 
@@ -25,7 +25,7 @@ gitCache=$( fn-git-cache "$BUILD_ROOT" "$BRANCH" )
 aggDir=$( fn-git-dir "$gitCache" "$AGGREGATOR_REPO" )
 
 if [ -z "$BUILD_ID" ]; then
-	BUILD_ID=$(fn-build-id "$BUILD_TYPE" )
+    BUILD_ID=$(fn-build-id "$BUILD_TYPE" )
 fi
 
 buildDirectory=$( fn-build-dir "$BUILD_ROOT" "$BUILD_ID" "$STREAM")
diff --git a/production/get-aggregator.sh b/production/get-aggregator.sh
index e177d7bfd..7a9b93d59 100644
--- a/production/get-aggregator.sh
+++ b/production/get-aggregator.sh
@@ -14,7 +14,7 @@ fi
 
 SCRIPT_PATH=${SCRIPT_PATH:-$(pwd)}
 
-source $SCRIPT_PATH/build-functions.sh
+source $SCRIPT_PATH/build-functions.shsource
 
 source "$1"
 
diff --git a/production/git-submodule-checkout.sh b/production/git-submodule-checkout.sh
index 2ff9aad0a..022472b95 100644
--- a/production/git-submodule-checkout.sh
+++ b/production/git-submodule-checkout.sh
@@ -6,29 +6,29 @@ name="$1"; shift
 
 VAL=$( grep "^${name}:" "$REPOSITORIES_TXT" | cut -f2 -d" ")
 
- # Here we count on $BUILD_TYPE being exported. TODO: make parameter later? 
+# Here we count on $BUILD_TYPE being exported. TODO: make parameter later? 
 if [[ -n "$BUILD_TYPE" && "$BUILD_TYPE" == "N" ]] 
 then
     if [[ "master" != $VAL ]]
     then
-       echo "INFO: Branch forced to 'master', instead of '$VAL', since doing N-Build"
-       VAL="master"
+        echo "INFO: Branch forced to 'master', instead of '$VAL', since doing N-Build"
+        VAL="master"
     fi
 fi
 
 
 if [ -z "$VAL" ]; then
-	echo No tag or branch specified for $name
-	exit
+    echo No tag or branch specified for $name
+    exit
 fi
 
 git fetch
 
 if [ -z "$(git tag -l $VAL)" ]; then
-	echo Updating branch $VAL
-	git checkout $VAL
-	git pull
+    echo Updating branch $VAL
+    git checkout $VAL
+    git pull
 else
-	echo Updating to tag $VAL
-	git checkout $VAL
+    echo Updating to tag $VAL
+    git checkout $VAL
 fi
diff --git a/production/master-build.sh b/production/master-build.sh
index 87e38ca37..7e96ee708 100644
--- a/production/master-build.sh
+++ b/production/master-build.sh
@@ -18,11 +18,12 @@ fi
 export SCRIPT_PATH="${BUILD_ROOT}/production"
 
 
-source "${SCRIPT_PATH}/build-functions.sh"
+source "${SCRIPT_PATH}/build-functions.shsource"
 
 source "${INITIAL_ENV_FILE}"
 
 
+
 cd $BUILD_ROOT
 
 buildrc=0
@@ -47,6 +48,14 @@ LOG=$buildDirectory/buildlogs/buildOutput.txt
 BUILD_PRETTY_DATE=$( date --date='@'$RAWDATE )
 TIMESTAMP=$( date +%Y%m%d-%H%M --date='@'$RAWDATE )
 
+# TRACE_OUTPUT is not normally used. But, it comes in handy for debugging
+# when output from some functions can not be written to stdout or stderr 
+# (due to the nature of the function ... it's "output" being its returned value).
+# When needed for local debugging, usually more convenient to provide 
+# a value relative to PWD in startup scripts. 
+export TRACE_OUTPUT=${TRACE_OUTPUT:-$buildDirectory/buildlogs/trace_output.txt}
+echo $BUILD_PRETTY_DATE > ${TRACE_OUTPUT}
+
 # These files have variable/value pairs for this build, suitable for use in 
 # shell scripts, PHP files, or as Ant (or Java) properties
 export BUILD_ENV_FILE=${buildDirectory}/buildproperties.shsource
diff --git a/production/parse-logs.sh b/production/parse-logs.sh
index 7ce6de8b6..a9102a1c1 100644
--- a/production/parse-logs.sh
+++ b/production/parse-logs.sh
@@ -2,19 +2,19 @@
 #
 
 if [ $# -ne 1 ]; then
-	echo USAGE: $0 env_file
-	exit 1
+    echo USAGE: $0 env_file
+    exit 1
 fi
 
 if [ ! -r "$1" ]; then
-	echo "$1" cannot be read
-	echo USAGE: $0 env_file
-	exit 1
+    echo "$1" cannot be read
+    echo USAGE: $0 env_file
+    exit 1
 fi
 
 SCRIPT_PATH=${SCRIPT_PATH:-$(pwd)}
 
-source $SCRIPT_PATH/build-functions.sh
+source $SCRIPT_PATH/build-functions.shsource
 
 source "$1"
 
@@ -26,7 +26,7 @@ gitCache=$( fn-git-cache "$BUILD_ROOT" "$BRANCH" )
 aggDir=$( fn-git-dir "$gitCache" "$AGGREGATOR_REPO" )
 
 if [ -z "$BUILD_ID" ]; then
-	BUILD_ID=$(fn-build-id "$BUILD_TYPE" )
+    BUILD_ID=$(fn-build-id "$BUILD_TYPE" )
 fi
 
 buildDirectory=$( fn-build-dir "$BUILD_ROOT" "$BUILD_ID" "${STREAM}" )
@@ -37,5 +37,5 @@ fn-checkout-basebuilder "$basebuilderDir" "$BASEBUILDER_TAG"
 launcherJar=$( fn-basebuilder-launcher "$basebuilderDir" )
 
 fn-parse-compile-logs "$BUILD_ID" \
-"$aggDir"/eclipse.platform.releng.tychoeclipsebuilder/eclipse/buildScripts/eclipse_convert.xml \
-"$buildDirectory" "$launcherJar"
+    "$aggDir"/eclipse.platform.releng.tychoeclipsebuilder/eclipse/buildScripts/eclipse_convert.xml \
+    "$buildDirectory" "$launcherJar"
diff --git a/production/parse-opts.sh b/production/parse-opts.sh
index 852335914..220bff0f5 100644
--- a/production/parse-opts.sh
+++ b/production/parse-opts.sh
@@ -7,15 +7,15 @@ set -- $( getopt -l buildArea:,stream:,branch: -o "" --  "$@" )
 
 
 while [ $# -gt 0 ]; do
-	case "$1" in
-		"--buildArea")
-			buildArea="$2"; shift;;
-		"--stream")
-			stream="$2"; shift;;
-		"--branch")
-			branch="$2"; shift;;
-	esac
-	shift
+    case "$1" in
+        "--buildArea")
+            buildArea="$2"; shift;;
+        "--stream")
+            stream="$2"; shift;;
+        "--branch")
+            branch="$2"; shift;;
+    esac
+    shift
 done
 
 buildArea_branch=$buildArea/$branch
@@ -23,6 +23,6 @@ gitCache=$buildArea_branch/gitCache
 
 
 if [ -r $gitCache/eclipse.platform.releng.aggregator ]; then
-	pushd $gitCache/eclipse.platform.releng.aggregator
+    pushd $gitCache/eclipse.platform.releng.aggregator
 else
 fi
diff --git a/production/pom-version-updater.sh b/production/pom-version-updater.sh
index 3a3a35957..9d7d248bb 100755
--- a/production/pom-version-updater.sh
+++ b/production/pom-version-updater.sh
@@ -14,7 +14,7 @@ fi
 
 SCRIPT_PATH=${SCRIPT_PATH:-$(pwd)}
 
-source $SCRIPT_PATH/build-functions.sh
+source $SCRIPT_PATH/build-functions.shsource
 
 source "$1"
 
diff --git a/production/promote-build.sh b/production/promote-build.sh
index 3fc263a7d..8220264fa 100755
--- a/production/promote-build.sh
+++ b/production/promote-build.sh
@@ -41,7 +41,7 @@ then
     SCRIPT_PATH=${PWD}
 fi
 
-source $SCRIPT_PATH/build-functions.sh
+source $SCRIPT_PATH/build-functions.shsource
 
 source "$2" 2>/dev/null
 
diff --git a/production/publish-eclipse.sh b/production/publish-eclipse.sh
index e687cefae..c336443cd 100644
--- a/production/publish-eclipse.sh
+++ b/production/publish-eclipse.sh
@@ -14,7 +14,7 @@ fi
 
 SCRIPT_PATH=${SCRIPT_PATH:-$(pwd)}
 
-source $SCRIPT_PATH/build-functions.sh
+source $SCRIPT_PATH/build-functions.shsource
 
 source "$1"
 
@@ -37,7 +37,7 @@ launcherJar=$( fn-basebuilder-launcher "$basebuilderDir" )
 
 fn-gather-compile-logs "$BUILD_ID" "$aggDir" "$buildDirectory"
 fn-parse-compile-logs "$BUILD_ID" \
-"$aggDir"/eclipse.platform.releng.tychoeclipsebuilder/eclipse/helper.xml \
-"$buildDirectory" "$launcherJar"
+    "$aggDir"/eclipse.platform.releng.tychoeclipsebuilder/eclipse/helper.xml \
+    "$buildDirectory" "$launcherJar"
 
 fn-publish-eclipse "$BUILD_TYPE" "$STREAM" "$BUILD_ID" "$aggDir" "$buildDirectory" "$launcherJar"
diff --git a/production/publish-equinox.sh b/production/publish-equinox.sh
index 4e84cdcfa..552ddb052 100644
--- a/production/publish-equinox.sh
+++ b/production/publish-equinox.sh
@@ -14,7 +14,7 @@ fi
 
 SCRIPT_PATH=${SCRIPT_PATH:-$(pwd)}
 
-source $SCRIPT_PATH/build-functions.sh
+source $SCRIPT_PATH/build-functions.shsource
 
 source "$1"
 
@@ -149,10 +149,10 @@ fn-checkout-basebuilder "$basebuilderDir" "$BASEBUILDER_TAG"
 
 launcherJar=$( fn-basebuilder-launcher "$basebuilderDir" )
 
-    fn-publish-equinox "$BUILD_TYPE" "$STREAM" "$BUILD_ID" "$aggDir" "$buildDirectory" "$launcherJar"
-    RC=$?
-    if [[ $RC != 0 ]] 
-    then
-        echo "ERROR: Somethign went wrong publishing Equinox. RC: $RC"
-        exit $RC
-    fi
+fn-publish-equinox "$BUILD_TYPE" "$STREAM" "$BUILD_ID" "$aggDir" "$buildDirectory" "$launcherJar"
+RC=$?
+if [[ $RC != 0 ]] 
+then
+    echo "ERROR: Somethign went wrong publishing Equinox. RC: $RC"
+    exit $RC
+fi
diff --git a/production/run-maven-build.sh b/production/run-maven-build.sh
index 710000e73..fabc119e5 100644
--- a/production/run-maven-build.sh
+++ b/production/run-maven-build.sh
@@ -14,7 +14,7 @@ fi
 
 SCRIPT_PATH=${SCRIPT_PATH:-$(pwd)}
 
-source $SCRIPT_PATH/build-functions.sh
+source $SCRIPT_PATH/build-functions.shsource
 
 source "$1"
 
diff --git a/production/sdk/relengUtils/re-master-build.sh b/production/sdk/relengUtils/re-master-build.sh
index 226c61881..26c4c4859 100755
--- a/production/sdk/relengUtils/re-master-build.sh
+++ b/production/sdk/relengUtils/re-master-build.sh
@@ -24,13 +24,15 @@ fi
 export SCRIPT_PATH="${BUILD_ROOT}/production"
 
 
-source "${SCRIPT_PATH}/build-functions.sh"
+source "${SCRIPT_PATH}/build-functions.shsource"
 
 source "${INITIAL_ENV_FILE}"
 
 
 cd $BUILD_ROOT
 
+buildrc=0
+
 # derived values
 
 
@@ -51,6 +53,14 @@ LOG=$buildDirectory/buildlogs/buildOutput.txt
 BUILD_PRETTY_DATE=$( date --date='@'$RAWDATE )
 TIMESTAMP=$( date +%Y%m%d-%H%M --date='@'$RAWDATE )
 
+# TRACE_OUTPUT is not normally used. But, it comes in handy for debugging
+# when output from some functions can not be written to stdout or stderr 
+# (due to the nature of the function ... it's "output" being its returned value).
+# When needed for local debugging, usually more convenient to provide 
+# a value relative to PWD in startup scripts. 
+export TRACE_OUTPUT=${TRACE_OUTPUT:-$buildDirectory/buildlogs/trace_output.txt}
+echo $BUILD_PRETTY_DATE > ${TRACE_OUTPUT}
+
 # These files have variable/value pairs for this build, suitable for use in 
 # shell scripts, PHP files, or as Ant (or Java) properties
 export BUILD_ENV_FILE=${buildDirectory}/buildproperties.shsource
@@ -128,49 +138,33 @@ echo "# Build ${BUILD_ID}, ${BUILD_PRETTY_DATE}" > ${buildDirectory}/directory.t
 
 
 
-    
-    pomUpdateFailed=false
+
+
 $SCRIPT_PATH/pom-version-updater.sh $BUILD_ENV_FILE 2>&1 | tee ${POM_VERSION_UPDATE_BUILD_LOG}
+# if file exists, pom update failed
 if [[ -f "${buildDirectory}/buildFailed-pom-version-updater" ]]
 then
-    pomUpdateFailed=true
+    buildrc=1
     /bin/grep "\[ERROR\]" "${POM_VERSION_UPDATE_BUILD_LOG}" >> "${buildDirectory}/buildFailed-pom-version-updater"
-fi
-    if $pomUpdateFailed 
-then 
-    # TODO: eventually put in more logic to "track" the failure, so
-    # proper actions and emails can be sent. For example, we'd still want to 
-    # publish what we have, but not start the tests.  
     echo "BUILD FAILED. See ${POM_VERSION_UPDATE_BUILD_LOG}." 
     BUILD_FAILED=${POM_VERSION_UPDATE_BUILD_LOG}
     fn-write-property BUILD_FAILED
-fi
-
-# if updater failed, something fairly large is wrong, so no need to compile
-    if ! $pomUpdateFailed 
-then
-    
+else
+    # if updater failed, something fairly large is wrong, so no need to compile
     $SCRIPT_PATH/run-maven-build.sh $BUILD_ENV_FILE 2>&1 | tee ${RUN_MAVEN_BUILD_LOG}
-    # does not seem be be "catching" error code via $?. Perhaps due to tee? 
-    # errors are "indicated" by special file
+    # if file exists, then run maven build failed.
     if [[ -f "${buildDirectory}/buildFailed-run-maven-build" ]]
     then
-        mavenBuildFailed=true
+        buildrc=1
         /bin/grep "\[ERROR\]" "${RUN_MAVEN_BUILD_LOG}" >> "${buildDirectory}/buildFailed-run-maven-build"
         BUILD_FAILED=${RUN_MAVEN_BUILD_LOG}
         fn-write-property BUILD_FAILED
-    fi
-    if [[ "${mavenBuildFailed}" ]] 
-    then 
         # TODO: eventually put in more logic to "track" the failure, so
         # proper actions and emails can be sent. For example, we'd still want to 
         # publish what we have, but not start the tests.  
         echo "BUILD FAILED. See ${RUN_MAVEN_BUILD_LOG}." 
-    fi
-
-    # if build failed, no need to gather parts
-    if [[ ! "${mavenBuildFailed}" ]] 
-    then 
+    else
+        # if build run maven build failed, no need to gather parts
         $SCRIPT_PATH/gather-parts.sh $BUILD_ENV_FILE 2>&1 | tee $logsDirectory/mb070_gather-parts_output.txt
         checkForErrorExit $? "Error occurred during gather parts"
     fi 
diff --git a/production/sdk/relengUtils/re-mb4I.sh b/production/sdk/relengUtils/re-mb4I.sh
index 388616200..95d514d3d 100755
--- a/production/sdk/relengUtils/re-mb4I.sh
+++ b/production/sdk/relengUtils/re-mb4I.sh
@@ -59,6 +59,8 @@ export BUILD_HOME=${BUILD_HOME:-/shared/eclipse/builds}
 # no override for minimal $PATH
 #export PATH=$JAVA_HOME/bin:$MAVEN_PATH:$ANT_HOME/bin:$PATH
 
+export TRACE_OUTPUT=${PWD}/trace_output.txt
+
 export BRANCH=master
 export BUILD_TYPE=I
 export STREAM=4.3.0
diff --git a/production/tag-build-input.sh b/production/tag-build-input.sh
index 8a4260335..d64cd470b 100644
--- a/production/tag-build-input.sh
+++ b/production/tag-build-input.sh
@@ -2,19 +2,19 @@
 #
 
 if [ $# -ne 1 ]; then
-	echo USAGE: $0 env_file
-	exit 1
+    echo USAGE: $0 env_file
+    exit 1
 fi
 
 if [ ! -r "$1" ]; then
-	echo "$1" cannot be read
-	echo USAGE: $0 env_file
-	exit 1
+    echo "$1" cannot be read
+    echo USAGE: $0 env_file
+    exit 1
 fi
 
 SCRIPT_PATH=${SCRIPT_PATH:-$(pwd)}
 
-source $SCRIPT_PATH/build-functions.sh
+source $SCRIPT_PATH/build-functions.shsource
 
 source "$1"
 
@@ -26,7 +26,7 @@ aggDir=$( fn-git-dir "$gitCache" "$AGGREGATOR_REPO" )
 repositories=$( echo $STREAMS_PATH/repositories.txt )
 
 if [ -z "$BUILD_ID" ]; then
-	BUILD_ID=$(fn-build-id "$BUILD_TYPE" )
+    BUILD_ID=$(fn-build-id "$BUILD_TYPE" )
 fi
 
 
diff --git a/production/update-build-input.sh b/production/update-build-input.sh
index 16b0608b1..465f178f1 100644
--- a/production/update-build-input.sh
+++ b/production/update-build-input.sh
@@ -2,19 +2,19 @@
 #
 
 if [ $# -ne 1 ]; then
-	echo USAGE: $0 env_file
-	exit 1
+    echo USAGE: $0 env_file
+    exit 1
 fi
 
 if [ ! -r "$1" ]; then
-	echo "$1" cannot be read
-	echo USAGE: $0 env_file
-	exit 1
+    echo "$1" cannot be read
+    echo USAGE: $0 env_file
+    exit 1
 fi
 
 SCRIPT_PATH=${SCRIPT_PATH:-$(pwd)}
 
-source $SCRIPT_PATH/build-functions.sh
+source $SCRIPT_PATH/build-functions.shsource
 
 source "$1"
 
@@ -29,7 +29,7 @@ repoScript=$( echo $SCRIPT_PATH/git-submodule-checkout.sh )
 
 
 if [ -z "$BUILD_ID" ]; then
-	BUILD_ID=$(fn-build-id "$BUILD_TYPE" )
+    BUILD_ID=$(fn-build-id "$BUILD_TYPE" )
 fi