releng.control/cleanupArtifacts.sh

#!/usr/bin/env bash


# remember to leave no slashes on filename in source command,
# (the commonVariations.shsource file, that is)
# so that users path is used to find it (first)
if [ -z $BUILD_INITIALIZED ]
then
   source commonVariations.shsource
   source ${RELENG_CONTROL}/commonComputedVariables.shsource
fi

# utility functions to remove directories over n days old

# We used to use atime (access time) so even if something was created/modified long ago, 
# if we read it often, such as for a "prereq", then we will know it is still in use. 
# Complication: I've learned that "access time" is modified even by things like ls or find, so 
# may not be that accurate if "still in use". We may want to change to "modified time" and 
# at same time change out
# build scripts to 'touch' prereq and basebuilder files each time they are used.
 
# Remember: for most funtions using times, 'n' time is essentially compared/converted to hours, 
# where hours = 24 + (n * 24), so basically ctime +0 means "at least 1 day old")

# = = = = = = = = = = = = = = = = = = = = = 


# requires parent Directiory as first argument
# if nDays not specified as second argument, then 0 is assumed 
#    (which is one day)
# if "saveAtLeast" is not set, as third argument, then assumed to be 1
#    (that is, at least "saveAtLeast" directories will be saved, no matter how old)
function removeOldArtifactsDirectories () 
{

parentDir=$1;
ndays=$2;
saveAtLeast=$3;

if [ -z $parentDir ]
then
   echo "ERROR: no parent directory specified in function call";
   return -1;
fi  

if [ -z ndays ] 
then 
   echo "INFO: ndays not specified, 1 day assumed";
   ndays=0
else 
   echo "INFO: ndays set to $ndays";   
fi

if [ -z saveAtLeast ] 
then 
   echo "INFO: saveAtLeast not specified, 1 assumed";
   saveAtLeast=0
else 
   echo "INFO: saveAtLeast set to $saveAtLeast";   
fi

echo;
echo "    Removing directories older than " $(( ${ndays} + 1 )) " days";
echo "        (from under ${parentDir})";
before=`find ${parentDir} -mindepth 2 -maxdepth 2 | wc -l`;
echo "            number of directories before cleaning: ${before}";

# empty directories often result from "bad builds". We remove those no matter how old
find ${parentDir} -mindepth 2 -maxdepth 3 -type d -empty -execdir rm -fr '{}' \;
# now remove old ones
find ${parentDir} -mindepth 2 -maxdepth 2 -type d -ctime +$ndays -execdir ${RELENG_CONTROL}/removeArtifactDirIf.sh '{}' $saveAtLeast \;

after=`find ${parentDir} -mindepth 2 -maxdepth 2 | wc -l`;
echo "            number of directories after cleaning: ${after}";
echo;

}

# requires parent Directiory as first argument
# if nDays not specified as second argument, then 0 is assumed 
#    (which is one day)

function removeOldDirectories () {

parentDir=$1;
ndays=$2;


if [ -z $parentDir ]
then
   echo "ERROR: no parent directory specified in function call";
   return -1;
fi  

if [ -z ndays ] 
then 
   echo "INFO: ndays not specified, 1 day assumed";
   ndays=0
else 
   echo "INFO: ndays set to $ndays";   
fi


echo;
echo "    Removing directories older  than " $(( ${ndays} + 1 )) " days";
echo "        (from under ${parentDir})";
before=`find ${parentDir} -mindepth 1 -maxdepth 1 | wc -l`;
echo "            number of directories before cleaning: ${before}";

# empty directories often result from "bad builds". We remove those no matter how old
find ${parentDir} -mindepth 1 -maxdepth 2 -type d -empty -execdir rm -fr '{}' \;
# now remove old ones
find ${parentDir} -mindepth 1 -maxdepth 1 -type d -ctime +$ndays -execdir ${RELENG_CONTROL}/removeIf.sh '{}' \;

after=`find ${parentDir} -mindepth 1 -maxdepth 1 | wc -l`;
echo "            number of directories after cleaning: ${after}";
echo;

}


# requires parent Directiory as first argument
# if nDays not specified as second argument, then 0 is assumed 
#    (which is one day)
# This is a good function to remove old files and directories from tmp
function removeOldDirectoriesAndFiles () 
{

parentDir=$1;
ndays=$2;


if [ -z $parentDir ]
then
   echo "ERROR: no parent directory specified in function call";
   return -1;
fi  

if [ -z ndays ] 
then 
   echo "INFO: ndays not specified, 14 days assumed";
   ndays=13
else 
   echo "INFO: ndays set to $ndays";   
fi


echo;
echo "    Removing directories and files older  than " $(( ${ndays} + 1 )) " days";
echo "        (from under ${parentDir})";
before=`find ${parentDir} | wc -l`;
echo "            number of directories and files before cleaning: ${before}";

# remove all old directories and files
find ${parentDir} -ctime +$ndays -execdir rm -fr '{}' \;

after=`find ${parentDir} | wc -l`;
echo "            number of directories and files after cleaning: ${after}";
echo;

}

# = = = = = = = = = = = = = = = = = = = = = 

# artifacts are special, since they have a three-layer directory structure, 
# and we want to be sure to always retain at least one, no matter how old. 
echo " ";
echo "INFO: Checking to remove old artifacts."
removeOldArtifactsDirectories ${PROJECT_ARTIFACTS} 3 1;

echo " ";
echo "INFO: Checking to remove old project working directories."
removeOldDirectories ${PROJECT_PROJECTS}  1;

echo " ";
echo "INFO: Checking to remove old testing directories."
removeOldDirectories ${PROJECT_TESTS} 1;

echo " ";
echo "INFO: Checking to remove old project builder directories."
removeOldDirectories ${PROJECT_BUILDERS} 3;

echo " ";
echo "INFO: Checking to remove old tmp directories and files."
removeOldDirectoriesAndFiles ${RECOMMENDED_TMP_DIR} 13;

# even once per month is a bit "risky" for prereqs, since some patch builds, for example, may not be ran except 
# once every blue moon ... hence pre-reqs will need to be re-fetched. And, remember, by that time, the 
# pre-req might have moved to a different URL (possibly archives), so our build scripts might have 
# to be changed to point to 'archive' location.
# TODO: we have some prereqs that can not be retrieved automatically. So, we should have 
# an "exclude" list, such as oagis_release_8.0_GA-withSP3.zip, DITA-OT1.2.1_bin-ASL.zip, WSDL_1.1_for%20OAGIS_8.0.zip.
# I don't recall which, but one or more of these have a "license confirmation" screen that prevents
# autodownload so when if/when we want it, we have to get our selves, and copy into prereqs directory.
# And one of those, I think oagis_release? has some write protected files that prevents their deletion, 
# once unzipped, so we have tweeked the permissions. 
echo " ";
echo "INFO: Checking to remove old pre-req directories and files.";
removeOldDirectoriesAndFiles ${LOCAL_PREREQS_CACHE} 30;

echo " ";
echo "INFO: Checking to remove old basebuilders.";
removeOldDirectories ${BASE_BUILDERS} 30;