diff options
| -rwxr-xr-x | cje-production/cleaners/cleanupNightlyRepo.sh | 213 | ||||
| -rwxr-xr-x | cje-production/cleaners/dailyCleanDownloads.sh | 126 |
2 files changed, 339 insertions, 0 deletions
diff --git a/cje-production/cleaners/cleanupNightlyRepo.sh b/cje-production/cleaners/cleanupNightlyRepo.sh new file mode 100755 index 000000000..d0f76fd1b --- /dev/null +++ b/cje-production/cleaners/cleanupNightlyRepo.sh @@ -0,0 +1,213 @@ +#!/bin/bash + +#******************************************************************************* +# Copyright (c) 2019 IBM Corporation and others. +# +# This program and the accompanying materials +# are made available under the terms of the Eclipse Public License 2.0 +# which accompanies this distribution, and is available at +# https://www.eclipse.org/legal/epl-2.0/ +# +# SPDX-License-Identifier: EPL-2.0 +# +# Contributors: +# IBM Corporation - initial API and implementation +#******************************************************************************* + +function writeHeader () +{ + compositeRepoDir="$1" + antBuildFile=$2 + if [[ -z "${compositeRepoDir}" ]] + then + echo -e "\n\tWARNING: compositeRepoDir not passed to writeHeader function as expected. But will continue with variable for later use?" + compositeRepoDir="\$\{compositeRepoDir\}" + fi + echo -e "<?xml version=\"1.0\" encoding=\"UTF-8\"?>" > $antBuildFile + echo -e "<project" >> $antBuildFile + echo -e " basedir=\".\"" >>$antBuildFile + echo -e " default=\"cleanup\">" >>$antBuildFile + echo -e " <target name=\"cleanup\">" >>$antBuildFile + echo -e " <p2.composite.repository>" >>$antBuildFile + echo -e " <repository location=\"file://${compositeRepoDir}\" />" >>$antBuildFile + echo -e " <remove>" >> $antBuildFile +} + +function writeReposToRemove () +{ + antBuildFile=$1 + for repo in "${reposToRemove[@]}" + do + echo " <repository location=\"$repo\" />" >> $antBuildFile + done + +} + +function writeClosing () +{ + antBuildFile=$1 + echo -e " </remove>" >> $antBuildFile + echo -e " </p2.composite.repository>" >> $antBuildFile + echo -e " </target>" >> $antBuildFile + echo -e "</project>" >> $antBuildFile +} + +function generateCleanupXML () +{ + mainRepoDir=$1 + antBuildFile=$2 + if [[ -z "${mainRepoDir}" || ! -e "${mainRepoDir}" ]] + then + echo -e "\n\tERROR: main repo to work with was not defined or did not exist" + else + writeHeader $mainRepoDir $antBuildFile + writeReposToRemove $antBuildFile + writeClosing $antBuildFile + fi +} + +function getReposToRemove () +{ + cDir="$1" + buildType=$2 + nRetain=$3 + + if [[ ! -e "${cDir}" ]] + then + echo -e "\n\tERROR: expected directory did not exist" >&2 + echo -e "\t\t${cDir}" >&2 + reposToRemove=() + return 1 + else + echo -e "\n\tDEBUG: working with directory ${cDir}" + # for "repo names" we want only the last segment of the directory, so use -printf %f. The %C@ is seconds since the beginning of time, for sorting. + # Some caution is needed here. Seems on eclipse.org "atime" is the one that reflects "when created", + # whereas ctime and mtime are all identical, in every directory?! Turns out, mine is that + # say too. Apparently p2 "touches" every directory, for some reason. Perhaps only in the "atomic" case? + # But, atime can vary from system to system, depending .. some systems do update, when accessed? + sortedallOldRepos=( $(find ${cDir} -maxdepth 1 -type d -name "${buildType}*" -printf "%C@ %f\n" | sort -n | cut -d\ -f2 ) ) + #nOldRepos=${#sortedallOldRepos[@]} + # all builds "find" command should match above, except for age related (and printf) arguments + nbuilds=$( find ${cDir} -maxdepth 1 -type d -name "${buildType}*" | wc -l ) + echo -e "\tNumber of repos before cleaning: $nbuilds" + #echo -e "\tNumber of old repos ${nOldRepos}" + echo -e "\tDEBUG contents of sortedallOldRepos array" + for item in "${sortedallOldRepos[@]}" + do + echo -e "\t${item}" + done + #totalMinusOld=$(( nbuilds - nOldRepos )) + #echo -e "\tDEBUG: total minus old: $totalMinusOld" + if [[ $nbuilds -gt $nRetain ]] + then + # remove all old ones, except for nRetain + nToRemove=$(( nbuilds - nRetain )) + echo -e "\tDEBUG: nToRemove: $nToRemove" + #remove all except newest nRetain (if more than nRetain) + if [[ ${nToRemove} -gt 0 ]] + then + echo -e "\tDEBUG: number of old repos to remove found to be ${nToRemove}" + reposToRemove=("${sortedallOldRepos[@]:0:$nToRemove}") + else + echo -e "\tDEBUG: number of old repos to remove found to be ${nToRemove} so we will remove none" + reposToRemove=() + fi + fi + fi +} + +function cleanRepo () +{ + eclipseRepo=$1 + buildType=$2 + nRetain=$3 + dryRun=$4 + # Changed to "hard coded" location of where to expect on Hudson. + eclipseexe=${workspace}/eclipse + if [[ ! -x ${eclipseexe} ]] + then + echo -e "\n\tERROR: expected eclipse location not found, or not executable" + echo -e "\t${eclipseexe}" + exit 1 + fi + javaexe=${workspace}/jdk8/jre/bin/java + if [[ ! -x ${javaexe} ]] + then + echo -e "\n\tERROR: expected java location not found, or not executable" + echo -e "\t${javaexe}" + exit 1 + fi + + antBuildFile=${workspace}/cleanupRepoScript${buildType}.xml + antRunner=org.eclipse.ant.core.antRunner + + # To allow this cron job to work from hudson, or traditional crontab + devWorkspace=${workspace}/workspace-cleanup + + echo -e "\tDEBUG: Cleaning repository ${eclipseRepo} on $HOSTNAME on $(date ) " >&2 + getReposToRemove "${eclipseRepo}" $buildType $nRetain + RC=$? + if [[ $RC == 0 ]] + then + # be sure there are some to remove + nToRemove=${#reposToRemove[@]} + if [[ $nToRemove == 0 ]] + then + echo -e "\tfound no files to remove for current repo" + else + echo -e "\n\tfound $nToRemove so generating ant file" + generateCleanupXML "${eclipseRepo}" $antBuildFile + if [[ -z "${dryRun}" ]] + then + $eclipseexe -nosplash --launcher.suppressErrors -data "${devWorkspace}" -application ${antRunner} -f $antBuildFile -vm ${javaexe} + RC=$? + fi + if [[ $RC == 0 ]] + then + # we only clean N-build directories, others need to be manually cleaned + # after every milestone, or after every release + if [[ $buildType == "N" ]] + then + for file in "${reposToRemove[@]}" + do + echo -e "\tDEBUG: directories to remove: ${eclipseRepo}/${file}" + if [[ -z "${dryRun}" ]] + then + rm -rf ${eclipseRepo}/${file} + fi + done + else + echo -e "\n\tReminder: only composite cleaned. For $buildType builds must cleanup simple repos ever milestone or release". + fi + fi + if [[ -n "${dryRun}" ]] + then + echo "Since dryrun printing $antBuildFile" + cat $antBuildFile + fi + fi + fi +} + + +workspace=$1 +remoteBase="/home/data/httpd/download.eclipse.org" + +eclipseIRepo="${remoteBase}/eclipse/updates/4.14-I-builds" +eclipseSRepo="${remoteBase}/eclipse/updates/4.14milestones" +eclipseYRepo="${remoteBase}/eclipse/updates/4.14-Y-builds" +eclipsePRepo="${remoteBase}/eclipse/updates/4.14-P-builds" + +#doDryrun=dryrun +doDryrun= +# global +declare -a reposToRemove=() +cleanRepo $eclipseIRepo I 4 $doDryrun +declare -a reposToRemove=() +cleanRepo $eclipseSRepo S 2 $doDryrun +declare -a reposToRemove=() +cleanRepo $eclipseYRepo I 2 $doDryrun +declare -a reposToRemove=() +cleanRepo $eclipsePRepo S 1 $doDryrun + +unset reposToRemove diff --git a/cje-production/cleaners/dailyCleanDownloads.sh b/cje-production/cleaners/dailyCleanDownloads.sh new file mode 100755 index 000000000..049be6180 --- /dev/null +++ b/cje-production/cleaners/dailyCleanDownloads.sh @@ -0,0 +1,126 @@ +#!/bin/bash +#******************************************************************************* +# Copyright (c) 2019 IBM Corporation and others. +# +# This program and the accompanying materials +# are made available under the terms of the Eclipse Public License 2.0 +# which accompanies this distribution, and is available at +# https://www.eclipse.org/legal/epl-2.0/ +# +# SPDX-License-Identifier: EPL-2.0 +# +# Contributors: +# Sravan Lakkimsetti - initial API and implementation +#******************************************************************************* + +# Utility to clean build machine +echo -e "\n\tDaily clean of ${HOSTNAME} download server on $(date )\n" + +# +# Checks whether a build can be retained or not. +# returns 0 if the build can be retained +# +function canBeRetained () +{ + retval=1 + buildName=$1 + grep BUILD_FAILED ${buildName}/buildproperties.shsource >/dev/null 2>&1 + if [ $? -ne 0 ]; then #build didn't fail + if [ -f "${buildName}/buildUnstable" ]; then #build is marked unstable so should not be retained + retval=1 + else + retval=0 + fi + fi + return $retval +} + +# +# remove a build +# + +function removeBuild () +{ + buildname=$1 + rm -fr $buildname + RC=$? + if [[ $RC = 0 ]];then + echo -e "Removed: $buildname" + else + echo -e "\n\tAn Error occurred removing $buildname. RC: $RC" + fi +} + +cDir="/home/data/httpd/download.eclipse.org/eclipse/downloads/drops4" +buildType="I*" +allOldBuilds=$( find ${cDir} -maxdepth 1 -type d -ctime +5 -name "${buildType}"|sort ) +nbuilds=$( find ${cDir} -maxdepth 1 -type d -name "${buildType}" | wc -l ) +echo -e "\tNumber of I-builds before cleaning: $nbuilds" + +# Make sure we leave at least 4 on DL server, no matter how old +# To avoid 'ls' see http://mywiki.wooledge.org/ParsingLs +# technically, applies to "find" as well. +shopt -s nullglob +count=0 +files=(${cDir}/${buildType}) +# We count on files being "ordered" by date/timestamp in name +for ((i=${#files[@]}-1; i>=0; --i)); do + newest[$count]="${files[$i]}" + count=$(( count + 1 )) + if [ $count -gt 3 ] + then + break + fi +done + +areNotToDelete=$(printf '%s\n' "${newest[@]}" | paste -sd '|') + +currentWeekNum=0 #week number from start of the year user to identify the week in which the build is created +found=0 #it will be 1 when we found a build that can be retained in that week + +for buildname in ${allOldBuilds}; do + if [[ $buildname =~ $areNotToDelete ]] + then + echo -e "\tDEBUG: Not removed (since one of 4 newest, even though old): \n\t$buildname" + else + buildId=$(basename $buildname) #extract buildId + yy=$(echo $buildId|cut -b2-5) #extract year from buildId + mm=$(echo $buildId|cut -b6-7) #extract month + dd=$(echo $buildId|cut -b8-9) #extract day + day=${mm}/${dd}/${yy} #construct build date + dayOfWeek=$(date -d $day +%u) #get the day of the week like monday, tue, etc + weekNum=$(date -d $day +%U) #get the week number from start of the year + + #special case for Sunday. unix considers sunday as the start of the week. but for us we need to consider + #monday as start of the week. For this purpose we subtract 1 to place the build in previous week + if [ $dayOfWeek -eq 7 ]; then + weekNum=$(expr $weekNum - 1) + if [ $weekNum -le 0 ]; then #check for the yearend + weekNum=53 + fi + fi + + canBeRetained ${buildname} + retain=$? + + if [ $weekNum -eq $currentWeekNum ]; then + if [ $retain -eq 0 -a $found -ne 1 ]; then # we didn't found a build that can be retained in the current week + found=1 + else + removeBuild $buildname + fi + else #week changed + currentWeekNum=$weekNum + found=0 + if [ $retain -eq 0 -a $dayOfWeek -eq 1 ]; then + found=1 + else + removeBuild $buildname + fi + fi + fi +done + +nbuilds=$( find ${cDir} -maxdepth 1 -type d -name "${buildType}" | wc -l ) +echo -e "\tNumber of N-builds after cleaning: $nbuilds" + |