Revision e6ec7e85 snf-tools/conf/snf-burnin-run.sh

b/snf-tools/conf/snf-burnin-run.sh
1 1
#!/bin/bash
2 2

  
3
#Example script for an snf-burnin cronjob.
4
#Starts an a snf-burnin test, deletes stale instances and archives old logs.
5
#It aborts if snf-burnin runs for longer than expected.
6

  
7
#Usage: ./snf-burnin-run.sh
8

  
9
API=""
10
PLANKTON=""
11
PLANKTON_USER=""
12
TOKEN=""
13
IMAGEID=""
14
OUTPUT=""
15

  
16
# Delete previously stalled objects.
17
snf-burnin --api=$API --token=$TOKEN --delete-stale
18

  
19
# Run burnin for 25 minutes. Fail after that.
20
timeout --foreground 25m \
21
    snf-burnin --api=$API \
22
               --plankton=$PLANKTON \
23
               --plankton-user=$PLANKTON_USER \
24
               --token=$TOKEN \
25
               --image-id=$IMAGEID \
26
               --action-timeout 120 \
27
               --log-folder=$OUTPUT \
28
               --nofailfast
29

  
30
# Delete possibly stalled objects.
31
snf-burnin --api=$API --token=$TOKEN --delete-stale
32

  
33
#Delete old folders
34
old=$(date -d "1 week ago" +%Y%m%d%H%M%S)
35
for dir in $OUTPUT/* ; do
36
    d=`basename $dir`
37
    (($d<$old)) && rm -r "$dir"
38
done
3

  
4
# --------------------------------------------------------------------
5
# Configure script parameters
6

  
7
# ----------------------------------------
8
# Here we define the tokens for each user burnin will
9
# test along with an alias for each token.
10
# For each user define an ALIAS, his TOKEN, an IMAGEID and a FLAVOR.
11
USERS=(\
12
    "burnin1" "token to be used" \
13
    "image id to be used" "flavor id to be used" \
14

  
15
    "burnin2" "token to be used" \
16
    "image id to be used" "flavor id to be used" \
17

  
18
    "burnin3" "token to be used" \
19
    "image id to be used" "flavor id to be used" \
20
  )
21

  
22
# ----------------------------------------
23
# Here we define the email parameters
24
# Email Tag
25
TAG="[synnefo.org-burnin]"
26
# Email Recipients
27
RECIPIENTS="burnin@synnefo.org"
28
# Subject for a successful burnin run
29
# (will be "$TAG ($ALIAS) $SUCCESS_SUBJECT" for each burnin instance)
30
SUCCESS_SUBJECT="Burnin Succeeded"
31
# Subject for a failed burnin run
32
# (will be "$TAG ($ALIAS) $FAILURE_SUBJECT" for each burnin instance)
33
FAILURE_SUBJECT="Burnin Failed"
34

  
35
# ----------------------------------------
36
# Some burnin parameters
37
AUTH_URL="https://accounts.synnefo.org/identity/v2.0/"
38
TIMEOUT=240
39

  
40
# ----------------------------------------
41
# Burnin executable and log files
42
Burnin="snf-burnin"
43
# Log Folder will be $LOGFOLDER/$ALIAS for each burnin instance
44
LOGFOLDER="/var/log/burnin_results/"
45
# Output file will be $OUTPUTFOLDER/burnin-$ALIAS.out for each burnin instance
46
OUTPUTFOLDER="/tmp"
47
# Lock file (we don't want two instances of this script)
48
LOCKFILE="/tmp/burnin.lockfile"
49

  
50

  
51
# --------------------------------------------------------------------
52
# Script functions
53

  
54
run_burnin() {
55
    local alias="$1"
56
    local token="$2"
57
    local image="$3"
58
    local flavor="$4"
59
    local success_subject="$TAG ($alias) $SUCCESS_SUBJECT"
60
    local failure_subject="$TAG ($alias) $FAILURE_SUBJECT"
61
    local logfolder="$LOGFOLDER/$alias"
62
    local outputfile="$OUTPUTFOLDER/burnin-$alias.out"
63
    local failed=false
64
    local error_summary
65
    local stale_subject
66

  
67
    # Save date-stamp to output
68
    date > $outputfile
69
    echo -e \
70
        "\n\n===== Burnin Output ========================================" \
71
        >> $outputfile
72

  
73
    # Check for stale servers/networks
74
    $Burnin --token="$token" --auth-url="$AUTH_URL" --show-stale 2>&1 | \
75
        grep "test" >> $outputfile 2>&1
76
    if [ $? -ne 0 ]; then
77
        # No stale servers/networks found. Run burnin
78
        $Burnin --token="$token" \
79
                --action-timeout="$TIMEOUT" \
80
                --image-id="$image" \
81
                --log-folder="$logfolder" \
82
                --auth-url="$AUTH_URL" \
83
                --force-flavor="$flavor" \
84
                --nofailfast \
85
            &>> $outputfile
86

  
87
        echo -e \
88
            "\n\n===== Burnin Logs ==========================================" \
89
            >> $outputfile
90

  
91
        # Search log files for errors
92
        for file in `ls -1d $logfolder/* | tail -1`/*/detail* ; do
93
            if egrep "(ERROR)|(FAILED)" $file > /dev/null; then
94
                failed=true
95
                echo "FILENAME: $file" >> $outputfile
96
                echo "ERROR: " >> $outputfile
97
                cat "$file" >> $outputfile
98
            fi
99
        done
100

  
101
        # Clean output file from escape characters
102
        sed -ri "s/\x1B\[([0-9]{1,2}(;[0-9]{1,2})?)?[m|K]//g" $outputfile
103

  
104
        # Send emails
105
        if $failed; then
106
            error_summary="`cat $outputfile | \
107
                egrep "ERROR: test_|FAIL: test_" | \
108
                awk '{ print $2 }' | grep -v "^$" | \
109
                sed -e 's/_/ /g' | cut -d" " -f3- | \
110
                tr '\n' ',' | sed -e 's/,$//g' | \
111
                sed -e 's/,/,  /g'`"
112
            cat $outputfile | /usr/bin/mailx -E \
113
                -s "$failure_subject: $error_summary" $RECIPIENTS
114
#        else
115
#            cat $outputfile | /usr/bin/mailx -E \
116
#                -s "$success_subject" $RECIPIENTS
117
        fi
118
    else
119
        # Burnin found stale servers/networks. Try to clean them
120
        $Burnin --token="$token" --auth-url="$AUTH_URL" --delete-stale \
121
            >> $outputfile 2>&1
122
        if [ $? -ne 0 ]; then
123
            stale_subject="$failure_subject: Couldn't delete stale servers/networks"
124
        else
125
            stale_subject="$success_subject: Stale servers/networks deleted"
126
        fi
127

  
128
        # Clean output file from escape characters
129
        sed -ri "s/\x1B\[([0-9]{1,2}(;[0-9]{1,2})?)?[m|K]//g" $outputfile
130

  
131
        # Send mail
132
        cat $outputfile | /usr/bin/mailx -E \
133
            -s "$stale_subject" $RECIPIENTS
134
    fi
135
}
136

  
137

  
138
# --------------------------------------------------------------------
139
# For each user run burnin function
140

  
141
(
142
    flock -xn 200 || exit 1
143

  
144
    set ${USERS[@]}
145

  
146
    while [ -n "$1" ]; do
147
        run_burnin "$1" "$2" "$3" "$4" &
148
        shift 4
149
    done
150

  
151
    wait
152
) 200>$LOCKFILE

Also available in: Unified diff