Revision f2e00c1f
b/snf-tools/conf/snf-burnin-run.sh | ||
---|---|---|
8 | 8 |
# Here we define the tokens for each user burnin will |
9 | 9 |
# test along with an alias for each token. |
10 | 10 |
# For each user define an ALIAS, his TOKEN, an IMAGEID and a FLAVOR. |
11 |
USERS=(\
|
|
12 |
"burnin1" "token to be used" \
|
|
13 |
"image id to be used" "flavor id to be used" \
|
|
11 |
USERS=( |
|
12 |
"burnin1" "token to be used" |
|
13 |
"name:Image name (reg expr)" "name:Flavor name (reg expr)"
|
|
14 | 14 |
|
15 |
"burnin2" "token to be used" \
|
|
16 |
"image id to be used" "flavor id to be used" \
|
|
15 |
"burnin2" "token to be used" |
|
16 |
"name:Image name (reg expr)" "name:Flavor name (reg expr)"
|
|
17 | 17 |
|
18 |
"burnin3" "token to be used" \
|
|
19 |
"image id to be used" "flavor id to be used" \
|
|
18 |
"burnin3" "token to be used" |
|
19 |
"name:Image name (reg expr)" "name:Flavor name (reg expr)"
|
|
20 | 20 |
) |
21 | 21 |
|
22 | 22 |
# ---------------------------------------- |
... | ... | |
35 | 35 |
# ---------------------------------------- |
36 | 36 |
# Some burnin parameters |
37 | 37 |
AUTH_URL="https://accounts.synnefo.org/identity/v2.0" |
38 |
SYSTEM_IMAGES_USER="uuid-of-owner-of-system-images" |
|
39 |
TIMEOUT=240 |
|
40 | 38 |
|
41 | 39 |
# ---------------------------------------- |
42 | 40 |
# Burnin executable and log files |
43 | 41 |
Burnin="snf-burnin" |
44 | 42 |
# Log Folder will be $LOGFOLDER/$ALIAS for each burnin instance |
45 |
LOGFOLDER="/var/log/burnin_results/" |
|
46 |
# Output file will be $OUTPUTFOLDER/burnin-$ALIAS.out for each burnin instance |
|
47 |
OUTPUTFOLDER="/tmp" |
|
43 |
LOGFOLDER="/var/log/burnin/" |
|
48 | 44 |
# Lock file (we don't want two instances of this script) |
49 | 45 |
LOCKFILE="/tmp/burnin.lockfile" |
50 | 46 |
|
... | ... | |
60 | 56 |
local success_subject="$TAG ($alias) $SUCCESS_SUBJECT" |
61 | 57 |
local failure_subject="$TAG ($alias) $FAILURE_SUBJECT" |
62 | 58 |
local logfolder="$LOGFOLDER/$alias" |
63 |
local outputfile="$OUTPUTFOLDER/burnin-$alias.out" |
|
64 |
local failed=false |
|
65 | 59 |
local error_summary |
66 | 60 |
local stale_subject |
67 | 61 |
|
68 |
# Save date-stamp to output |
|
69 |
date > $outputfile |
|
70 |
echo -e \ |
|
71 |
"\n\n===== Burnin Output ========================================" \ |
|
72 |
>> $outputfile |
|
73 |
|
|
74 | 62 |
# Check for stale servers/networks |
75 |
$Burnin --token="$token" --auth-url="$AUTH_URL" --show-stale 2>&1 | \ |
|
76 |
grep "test" >> $outputfile 2>&1 |
|
77 |
if [ $? -ne 0 ]; then |
|
63 |
$Burnin --token="$token" --auth-url="$AUTH_URL" --show-stale --quiet |
|
64 |
if [ $? -eq 0 ]; then |
|
78 | 65 |
# No stale servers/networks found. Run burnin |
79 |
$Burnin --token="$token" \ |
|
80 |
--action-timeout="$TIMEOUT" \ |
|
81 |
--image-id="$image" \ |
|
82 |
--log-folder="$logfolder" \ |
|
83 |
--auth-url="$AUTH_URL" \ |
|
84 |
--force-flavor="$flavor" \ |
|
85 |
--system-images-user="$SYSTEM_IMAGES_USER" \ |
|
86 |
--nofailfast \ |
|
87 |
&>> $outputfile |
|
88 |
|
|
89 |
echo -e \ |
|
90 |
"\n\n===== Burnin Logs ==========================================" \ |
|
91 |
>> $outputfile |
|
92 |
|
|
93 |
# Search log files for errors |
|
94 |
for file in `ls -1d $logfolder/* | tail -1`/*/detail* ; do |
|
95 |
if egrep "(ERROR)|(FAILED)" $file > /dev/null; then |
|
96 |
failed=true |
|
97 |
echo "FILENAME: $file" >> $outputfile |
|
98 |
echo "ERROR: " >> $outputfile |
|
99 |
cat "$file" >> $outputfile |
|
100 |
fi |
|
101 |
done |
|
102 |
|
|
103 |
# Clean output file from escape characters |
|
104 |
sed -ri "s/\x1B\[([0-9]{1,2}(;[0-9]{1,2})?)?[m|K]//g" $outputfile |
|
105 |
|
|
106 |
# Send emails |
|
107 |
if $failed; then |
|
108 |
error_summary="`cat $outputfile | \ |
|
109 |
egrep "ERROR: test_|FAIL: test_" | \ |
|
110 |
awk '{ print $2 }' | grep -v "^$" | \ |
|
111 |
sed -e 's/_/ /g' | cut -d" " -f3- | \ |
|
112 |
tr '\n' ',' | sed -e 's/,$//g' | \ |
|
113 |
sed -e 's/,/, /g'`" |
|
114 |
cat $outputfile | /usr/bin/mailx -E \ |
|
66 |
results=$($Burnin \ |
|
67 |
--token="$token" \ |
|
68 |
--auth-url="$AUTH_URL" \ |
|
69 |
--images="$image" \ |
|
70 |
--flavors="$flavor" \ |
|
71 |
--log-folder="$logfolder" \ |
|
72 |
--final-report-only \ |
|
73 |
2>&1) |
|
74 |
|
|
75 |
if [ $? -ne 0 ]; then |
|
76 |
# Burnin failed |
|
77 |
# Send email |
|
78 |
error_summary=$(echo "$results" | \ |
|
79 |
sed -n 's/ \* Failed: \(.*\)/\1/p') |
|
80 |
echo "$results" | /usr/bin/mailx -E \ |
|
115 | 81 |
-s "$failure_subject: $error_summary" $RECIPIENTS |
116 | 82 |
# else |
117 |
# cat $outputfile | /usr/bin/mailx -E \
|
|
83 |
# echo "$results" | /usr/bin/mailx -E \
|
|
118 | 84 |
# -s "$success_subject" $RECIPIENTS |
119 | 85 |
fi |
120 | 86 |
else |
121 | 87 |
# Burnin found stale servers/networks. Try to clean them |
122 |
$Burnin --token="$token" --auth-url="$AUTH_URL" --delete-stale \
|
|
123 |
>> $outputfile 2>&1
|
|
88 |
results=$($Burnin --token="$token" --auth-url="$AUTH_URL" \
|
|
89 |
--delete-stale --log-folder="$logfolder" --final-report-only 2>&1)
|
|
124 | 90 |
if [ $? -ne 0 ]; then |
125 | 91 |
stale_subject="$failure_subject: Couldn't delete stale servers/networks" |
126 | 92 |
else |
127 | 93 |
stale_subject="$success_subject: Stale servers/networks deleted" |
128 | 94 |
fi |
129 | 95 |
|
130 |
# Clean output file from escape characters |
|
131 |
sed -ri "s/\x1B\[([0-9]{1,2}(;[0-9]{1,2})?)?[m|K]//g" $outputfile |
|
132 |
|
|
133 | 96 |
# Send mail |
134 |
cat $outputfile | /usr/bin/mailx -E \
|
|
97 |
echo "$results" | /usr/bin/mailx -E \
|
|
135 | 98 |
-s "$stale_subject" $RECIPIENTS |
136 | 99 |
fi |
137 | 100 |
} |
... | ... | |
143 | 106 |
( |
144 | 107 |
flock -xn 200 || exit 1 |
145 | 108 |
|
146 |
set ${USERS[@]}
|
|
109 |
set "${USERS[@]}"
|
|
147 | 110 |
|
148 | 111 |
while [ -n "$1" ]; do |
149 | 112 |
run_burnin "$1" "$2" "$3" "$4" & |
Also available in: Unified diff