Revision e6ec7e85 snf-tools/conf/snf-burnin-run.sh
b/snf-tools/conf/snf-burnin-run.sh | ||
---|---|---|
1 | 1 |
#!/bin/bash |
2 | 2 |
|
3 |
#Example script for an snf-burnin cronjob. |
|
4 |
#Starts an a snf-burnin test, deletes stale instances and archives old logs. |
|
5 |
#It aborts if snf-burnin runs for longer than expected. |
|
6 |
|
|
7 |
#Usage: ./snf-burnin-run.sh |
|
8 |
|
|
9 |
API="" |
|
10 |
PLANKTON="" |
|
11 |
PLANKTON_USER="" |
|
12 |
TOKEN="" |
|
13 |
IMAGEID="" |
|
14 |
OUTPUT="" |
|
15 |
|
|
16 |
# Delete previously stalled objects. |
|
17 |
snf-burnin --api=$API --token=$TOKEN --delete-stale |
|
18 |
|
|
19 |
# Run burnin for 25 minutes. Fail after that. |
|
20 |
timeout --foreground 25m \ |
|
21 |
snf-burnin --api=$API \ |
|
22 |
--plankton=$PLANKTON \ |
|
23 |
--plankton-user=$PLANKTON_USER \ |
|
24 |
--token=$TOKEN \ |
|
25 |
--image-id=$IMAGEID \ |
|
26 |
--action-timeout 120 \ |
|
27 |
--log-folder=$OUTPUT \ |
|
28 |
--nofailfast |
|
29 |
|
|
30 |
# Delete possibly stalled objects. |
|
31 |
snf-burnin --api=$API --token=$TOKEN --delete-stale |
|
32 |
|
|
33 |
#Delete old folders |
|
34 |
old=$(date -d "1 week ago" +%Y%m%d%H%M%S) |
|
35 |
for dir in $OUTPUT/* ; do |
|
36 |
d=`basename $dir` |
|
37 |
(($d<$old)) && rm -r "$dir" |
|
38 |
done |
|
3 |
|
|
4 |
# -------------------------------------------------------------------- |
|
5 |
# Configure script parameters |
|
6 |
|
|
7 |
# ---------------------------------------- |
|
8 |
# Here we define the tokens for each user burnin will |
|
9 |
# test along with an alias for each token. |
|
10 |
# For each user define an ALIAS, his TOKEN, an IMAGEID and a FLAVOR. |
|
11 |
USERS=(\ |
|
12 |
"burnin1" "token to be used" \ |
|
13 |
"image id to be used" "flavor id to be used" \ |
|
14 |
|
|
15 |
"burnin2" "token to be used" \ |
|
16 |
"image id to be used" "flavor id to be used" \ |
|
17 |
|
|
18 |
"burnin3" "token to be used" \ |
|
19 |
"image id to be used" "flavor id to be used" \ |
|
20 |
) |
|
21 |
|
|
22 |
# ---------------------------------------- |
|
23 |
# Here we define the email parameters |
|
24 |
# Email Tag |
|
25 |
TAG="[synnefo.org-burnin]" |
|
26 |
# Email Recipients |
|
27 |
RECIPIENTS="burnin@synnefo.org" |
|
28 |
# Subject for a successful burnin run |
|
29 |
# (will be "$TAG ($ALIAS) $SUCCESS_SUBJECT" for each burnin instance) |
|
30 |
SUCCESS_SUBJECT="Burnin Succeeded" |
|
31 |
# Subject for a failed burnin run |
|
32 |
# (will be "$TAG ($ALIAS) $FAILURE_SUBJECT" for each burnin instance) |
|
33 |
FAILURE_SUBJECT="Burnin Failed" |
|
34 |
|
|
35 |
# ---------------------------------------- |
|
36 |
# Some burnin parameters |
|
37 |
AUTH_URL="https://accounts.synnefo.org/identity/v2.0/" |
|
38 |
TIMEOUT=240 |
|
39 |
|
|
40 |
# ---------------------------------------- |
|
41 |
# Burnin executable and log files |
|
42 |
Burnin="snf-burnin" |
|
43 |
# Log Folder will be $LOGFOLDER/$ALIAS for each burnin instance |
|
44 |
LOGFOLDER="/var/log/burnin_results/" |
|
45 |
# Output file will be $OUTPUTFOLDER/burnin-$ALIAS.out for each burnin instance |
|
46 |
OUTPUTFOLDER="/tmp" |
|
47 |
# Lock file (we don't want two instances of this script) |
|
48 |
LOCKFILE="/tmp/burnin.lockfile" |
|
49 |
|
|
50 |
|
|
51 |
# -------------------------------------------------------------------- |
|
52 |
# Script functions |
|
53 |
|
|
54 |
run_burnin() { |
|
55 |
local alias="$1" |
|
56 |
local token="$2" |
|
57 |
local image="$3" |
|
58 |
local flavor="$4" |
|
59 |
local success_subject="$TAG ($alias) $SUCCESS_SUBJECT" |
|
60 |
local failure_subject="$TAG ($alias) $FAILURE_SUBJECT" |
|
61 |
local logfolder="$LOGFOLDER/$alias" |
|
62 |
local outputfile="$OUTPUTFOLDER/burnin-$alias.out" |
|
63 |
local failed=false |
|
64 |
local error_summary |
|
65 |
local stale_subject |
|
66 |
|
|
67 |
# Save date-stamp to output |
|
68 |
date > $outputfile |
|
69 |
echo -e \ |
|
70 |
"\n\n===== Burnin Output ========================================" \ |
|
71 |
>> $outputfile |
|
72 |
|
|
73 |
# Check for stale servers/networks |
|
74 |
$Burnin --token="$token" --auth-url="$AUTH_URL" --show-stale 2>&1 | \ |
|
75 |
grep "test" >> $outputfile 2>&1 |
|
76 |
if [ $? -ne 0 ]; then |
|
77 |
# No stale servers/networks found. Run burnin |
|
78 |
$Burnin --token="$token" \ |
|
79 |
--action-timeout="$TIMEOUT" \ |
|
80 |
--image-id="$image" \ |
|
81 |
--log-folder="$logfolder" \ |
|
82 |
--auth-url="$AUTH_URL" \ |
|
83 |
--force-flavor="$flavor" \ |
|
84 |
--nofailfast \ |
|
85 |
&>> $outputfile |
|
86 |
|
|
87 |
echo -e \ |
|
88 |
"\n\n===== Burnin Logs ==========================================" \ |
|
89 |
>> $outputfile |
|
90 |
|
|
91 |
# Search log files for errors |
|
92 |
for file in `ls -1d $logfolder/* | tail -1`/*/detail* ; do |
|
93 |
if egrep "(ERROR)|(FAILED)" $file > /dev/null; then |
|
94 |
failed=true |
|
95 |
echo "FILENAME: $file" >> $outputfile |
|
96 |
echo "ERROR: " >> $outputfile |
|
97 |
cat "$file" >> $outputfile |
|
98 |
fi |
|
99 |
done |
|
100 |
|
|
101 |
# Clean output file from escape characters |
|
102 |
sed -ri "s/\x1B\[([0-9]{1,2}(;[0-9]{1,2})?)?[m|K]//g" $outputfile |
|
103 |
|
|
104 |
# Send emails |
|
105 |
if $failed; then |
|
106 |
error_summary="`cat $outputfile | \ |
|
107 |
egrep "ERROR: test_|FAIL: test_" | \ |
|
108 |
awk '{ print $2 }' | grep -v "^$" | \ |
|
109 |
sed -e 's/_/ /g' | cut -d" " -f3- | \ |
|
110 |
tr '\n' ',' | sed -e 's/,$//g' | \ |
|
111 |
sed -e 's/,/, /g'`" |
|
112 |
cat $outputfile | /usr/bin/mailx -E \ |
|
113 |
-s "$failure_subject: $error_summary" $RECIPIENTS |
|
114 |
# else |
|
115 |
# cat $outputfile | /usr/bin/mailx -E \ |
|
116 |
# -s "$success_subject" $RECIPIENTS |
|
117 |
fi |
|
118 |
else |
|
119 |
# Burnin found stale servers/networks. Try to clean them |
|
120 |
$Burnin --token="$token" --auth-url="$AUTH_URL" --delete-stale \ |
|
121 |
>> $outputfile 2>&1 |
|
122 |
if [ $? -ne 0 ]; then |
|
123 |
stale_subject="$failure_subject: Couldn't delete stale servers/networks" |
|
124 |
else |
|
125 |
stale_subject="$success_subject: Stale servers/networks deleted" |
|
126 |
fi |
|
127 |
|
|
128 |
# Clean output file from escape characters |
|
129 |
sed -ri "s/\x1B\[([0-9]{1,2}(;[0-9]{1,2})?)?[m|K]//g" $outputfile |
|
130 |
|
|
131 |
# Send mail |
|
132 |
cat $outputfile | /usr/bin/mailx -E \ |
|
133 |
-s "$stale_subject" $RECIPIENTS |
|
134 |
fi |
|
135 |
} |
|
136 |
|
|
137 |
|
|
138 |
# -------------------------------------------------------------------- |
|
139 |
# For each user run burnin function |
|
140 |
|
|
141 |
( |
|
142 |
flock -xn 200 || exit 1 |
|
143 |
|
|
144 |
set ${USERS[@]} |
|
145 |
|
|
146 |
while [ -n "$1" ]; do |
|
147 |
run_burnin "$1" "$2" "$3" "$4" & |
|
148 |
shift 4 |
|
149 |
done |
|
150 |
|
|
151 |
wait |
|
152 |
) 200>$LOCKFILE |
Also available in: Unified diff