commit e6ec7e85fae368dbde658ca4159a669077c5a553
Author: Ilias Tsitsimpis <iliastsi@grnet.gr>
Date:   Thu Jun 20 13:50:23 2013 +0300

    burnin: Add new bash script to run burnin

diff --git a/snf-tools/conf/snf-burnin-run.sh b/snf-tools/conf/snf-burnin-run.sh
index f2bea73..76497b1 100644
--- a/snf-tools/conf/snf-burnin-run.sh
+++ b/snf-tools/conf/snf-burnin-run.sh
@@ -1,38 +1,152 @@
 #!/bin/bash
 
-#Example script for an snf-burnin cronjob.
-#Starts an a snf-burnin test, deletes stale instances and archives old logs.
-#It aborts if snf-burnin runs for longer than expected.
-
-#Usage: ./snf-burnin-run.sh
-
-API=""
-PLANKTON=""
-PLANKTON_USER=""
-TOKEN=""
-IMAGEID=""
-OUTPUT=""
-
-# Delete previously stalled objects.
-snf-burnin --api=$API --token=$TOKEN --delete-stale
-
-# Run burnin for 25 minutes. Fail after that.
-timeout --foreground 25m \
-    snf-burnin --api=$API \
-               --plankton=$PLANKTON \
-               --plankton-user=$PLANKTON_USER \
-               --token=$TOKEN \
-               --image-id=$IMAGEID \
-               --action-timeout 120 \
-               --log-folder=$OUTPUT \
-               --nofailfast
-
-# Delete possibly stalled objects.
-snf-burnin --api=$API --token=$TOKEN --delete-stale
-
-#Delete old folders
-old=$(date -d "1 week ago" +%Y%m%d%H%M%S)
-for dir in $OUTPUT/* ; do
-    d=`basename $dir`
-    (($d<$old)) && rm -r "$dir"
-done
+
+# --------------------------------------------------------------------
+# Configure script parameters
+
+# ----------------------------------------
+# Here we define the tokens for each user burnin will
+# test along with an alias for each token.
+# For each user define an ALIAS, his TOKEN, an IMAGEID and a FLAVOR.
+USERS=(\
+    "burnin1" "token to be used" \
+    "image id to be used" "flavor id to be used" \
+
+    "burnin2" "token to be used" \
+    "image id to be used" "flavor id to be used" \
+
+    "burnin3" "token to be used" \
+    "image id to be used" "flavor id to be used" \
+  )
+
+# ----------------------------------------
+# Here we define the email parameters
+# Email Tag
+TAG="[synnefo.org-burnin]"
+# Email Recipients
+RECIPIENTS="burnin@synnefo.org"
+# Subject for a successful burnin run
+# (will be "$TAG ($ALIAS) $SUCCESS_SUBJECT" for each burnin instance)
+SUCCESS_SUBJECT="Burnin Succeeded"
+# Subject for a failed burnin run
+# (will be "$TAG ($ALIAS) $FAILURE_SUBJECT" for each burnin instance)
+FAILURE_SUBJECT="Burnin Failed"
+
+# ----------------------------------------
+# Some burnin parameters
+AUTH_URL="https://accounts.synnefo.org/identity/v2.0/"
+TIMEOUT=240
+
+# ----------------------------------------
+# Burnin executable and log files
+Burnin="snf-burnin"
+# Log Folder will be $LOGFOLDER/$ALIAS for each burnin instance
+LOGFOLDER="/var/log/burnin_results/"
+# Output file will be $OUTPUTFOLDER/burnin-$ALIAS.out for each burnin instance
+OUTPUTFOLDER="/tmp"
+# Lock file (we don't want two instances of this script)
+LOCKFILE="/tmp/burnin.lockfile"
+
+
+# --------------------------------------------------------------------
+# Script functions
+
+run_burnin() {
+    local alias="$1"
+    local token="$2"
+    local image="$3"
+    local flavor="$4"
+    local success_subject="$TAG ($alias) $SUCCESS_SUBJECT"
+    local failure_subject="$TAG ($alias) $FAILURE_SUBJECT"
+    local logfolder="$LOGFOLDER/$alias"
+    local outputfile="$OUTPUTFOLDER/burnin-$alias.out"
+    local failed=false
+    local error_summary
+    local stale_subject
+
+    # Save date-stamp to output
+    date > $outputfile
+    echo -e \
+        "\n\n===== Burnin Output ========================================" \
+        >> $outputfile
+
+    # Check for stale servers/networks
+    $Burnin --token="$token" --auth-url="$AUTH_URL" --show-stale 2>&1 | \
+        grep "test" >> $outputfile 2>&1
+    if [ $? -ne 0 ]; then
+        # No stale servers/networks found. Run burnin
+        $Burnin --token="$token" \
+                --action-timeout="$TIMEOUT" \
+                --image-id="$image" \
+                --log-folder="$logfolder" \
+                --auth-url="$AUTH_URL" \
+                --force-flavor="$flavor" \
+                --nofailfast \
+            &>> $outputfile
+
+        echo -e \
+            "\n\n===== Burnin Logs ==========================================" \
+            >> $outputfile
+
+        # Search log files for errors
+        for file in `ls -1d $logfolder/* | tail -1`/*/detail* ; do
+            if egrep "(ERROR)|(FAILED)" $file > /dev/null; then
+                failed=true
+                echo "FILENAME: $file" >> $outputfile
+                echo "ERROR: " >> $outputfile
+                cat "$file" >> $outputfile
+            fi
+        done
+
+        # Clean output file from escape characters
+        sed -ri "s/\x1B\[([0-9]{1,2}(;[0-9]{1,2})?)?[m|K]//g" $outputfile
+
+        # Send emails
+        if $failed; then
+            error_summary="`cat $outputfile | \
+                egrep "ERROR: test_|FAIL: test_" | \
+                awk '{ print $2 }' | grep -v "^$" | \
+                sed -e 's/_/ /g' | cut -d" " -f3- | \
+                tr '\n' ',' | sed -e 's/,$//g' | \
+                sed -e 's/,/,  /g'`"
+            cat $outputfile | /usr/bin/mailx -E \
+                -s "$failure_subject: $error_summary" $RECIPIENTS
+#        else
+#            cat $outputfile | /usr/bin/mailx -E \
+#                -s "$success_subject" $RECIPIENTS
+        fi
+    else
+        # Burnin found stale servers/networks. Try to clean them
+        $Burnin --token="$token" --auth-url="$AUTH_URL" --delete-stale \
+            >> $outputfile 2>&1
+        if [ $? -ne 0 ]; then
+            stale_subject="$failure_subject: Couldn't delete stale servers/networks"
+        else
+            stale_subject="$success_subject: Stale servers/networks deleted"
+        fi
+
+        # Clean output file from escape characters
+        sed -ri "s/\x1B\[([0-9]{1,2}(;[0-9]{1,2})?)?[m|K]//g" $outputfile
+
+        # Send mail
+        cat $outputfile | /usr/bin/mailx -E \
+            -s "$stale_subject" $RECIPIENTS
+    fi
+}
+
+
+# --------------------------------------------------------------------
+# For each user run burnin function
+
+(
+    flock -xn 200 || exit 1
+
+    set ${USERS[@]}
+
+    while [ -n "$1" ]; do
+        run_burnin "$1" "$2" "$3" "$4" &
+        shift 4
+    done
+
+    wait
+) 200>$LOCKFILE
