root / snf-tools / conf / snf-burnin-run.sh @ faad3c72
History | View | Annotate | Download (4.9 kB)
1 | 076a2dc0 | Ilias Tsitsimpis | #!/bin/bash |
---|---|---|---|
2 | 56d5c612 | John Giannelos | |
3 | e6ec7e85 | Ilias Tsitsimpis | |
4 | e6ec7e85 | Ilias Tsitsimpis | # -------------------------------------------------------------------- |
5 | e6ec7e85 | Ilias Tsitsimpis | # Configure script parameters |
6 | e6ec7e85 | Ilias Tsitsimpis | |
7 | e6ec7e85 | Ilias Tsitsimpis | # ---------------------------------------- |
8 | e6ec7e85 | Ilias Tsitsimpis | # Here we define the tokens for each user burnin will |
9 | e6ec7e85 | Ilias Tsitsimpis | # test along with an alias for each token. |
10 | e6ec7e85 | Ilias Tsitsimpis | # For each user define an ALIAS, his TOKEN, an IMAGEID and a FLAVOR. |
11 | e6ec7e85 | Ilias Tsitsimpis | USERS=(\ |
12 | e6ec7e85 | Ilias Tsitsimpis | "burnin1" "token to be used" \ |
13 | e6ec7e85 | Ilias Tsitsimpis | "image id to be used" "flavor id to be used" \ |
14 | e6ec7e85 | Ilias Tsitsimpis | |
15 | e6ec7e85 | Ilias Tsitsimpis | "burnin2" "token to be used" \ |
16 | e6ec7e85 | Ilias Tsitsimpis | "image id to be used" "flavor id to be used" \ |
17 | e6ec7e85 | Ilias Tsitsimpis | |
18 | e6ec7e85 | Ilias Tsitsimpis | "burnin3" "token to be used" \ |
19 | e6ec7e85 | Ilias Tsitsimpis | "image id to be used" "flavor id to be used" \ |
20 | e6ec7e85 | Ilias Tsitsimpis | ) |
21 | e6ec7e85 | Ilias Tsitsimpis | |
22 | e6ec7e85 | Ilias Tsitsimpis | # ---------------------------------------- |
23 | e6ec7e85 | Ilias Tsitsimpis | # Here we define the email parameters |
24 | e6ec7e85 | Ilias Tsitsimpis | # Email Tag |
25 | e6ec7e85 | Ilias Tsitsimpis | TAG="[synnefo.org-burnin]" |
26 | e6ec7e85 | Ilias Tsitsimpis | # Email Recipients |
27 | e6ec7e85 | Ilias Tsitsimpis | RECIPIENTS="burnin@synnefo.org" |
28 | e6ec7e85 | Ilias Tsitsimpis | # Subject for a successful burnin run |
29 | e6ec7e85 | Ilias Tsitsimpis | # (will be "$TAG ($ALIAS) $SUCCESS_SUBJECT" for each burnin instance) |
30 | e6ec7e85 | Ilias Tsitsimpis | SUCCESS_SUBJECT="Burnin Succeeded" |
31 | e6ec7e85 | Ilias Tsitsimpis | # Subject for a failed burnin run |
32 | e6ec7e85 | Ilias Tsitsimpis | # (will be "$TAG ($ALIAS) $FAILURE_SUBJECT" for each burnin instance) |
33 | e6ec7e85 | Ilias Tsitsimpis | FAILURE_SUBJECT="Burnin Failed" |
34 | e6ec7e85 | Ilias Tsitsimpis | |
35 | e6ec7e85 | Ilias Tsitsimpis | # ---------------------------------------- |
36 | e6ec7e85 | Ilias Tsitsimpis | # Some burnin parameters |
37 | e6ec7e85 | Ilias Tsitsimpis | AUTH_URL="https://accounts.synnefo.org/identity/v2.0/" |
38 | 33d1cf90 | Ilias Tsitsimpis | SYSTEM_IMAGES_USER="uuid-of-owner-of-system-images" |
39 | e6ec7e85 | Ilias Tsitsimpis | TIMEOUT=240 |
40 | e6ec7e85 | Ilias Tsitsimpis | |
41 | e6ec7e85 | Ilias Tsitsimpis | # ---------------------------------------- |
42 | e6ec7e85 | Ilias Tsitsimpis | # Burnin executable and log files |
43 | e6ec7e85 | Ilias Tsitsimpis | Burnin="snf-burnin" |
44 | e6ec7e85 | Ilias Tsitsimpis | # Log Folder will be $LOGFOLDER/$ALIAS for each burnin instance |
45 | e6ec7e85 | Ilias Tsitsimpis | LOGFOLDER="/var/log/burnin_results/" |
46 | e6ec7e85 | Ilias Tsitsimpis | # Output file will be $OUTPUTFOLDER/burnin-$ALIAS.out for each burnin instance |
47 | e6ec7e85 | Ilias Tsitsimpis | OUTPUTFOLDER="/tmp" |
48 | e6ec7e85 | Ilias Tsitsimpis | # Lock file (we don't want two instances of this script) |
49 | e6ec7e85 | Ilias Tsitsimpis | LOCKFILE="/tmp/burnin.lockfile" |
50 | e6ec7e85 | Ilias Tsitsimpis | |
51 | e6ec7e85 | Ilias Tsitsimpis | |
52 | e6ec7e85 | Ilias Tsitsimpis | # -------------------------------------------------------------------- |
53 | e6ec7e85 | Ilias Tsitsimpis | # Script functions |
54 | e6ec7e85 | Ilias Tsitsimpis | |
55 | e6ec7e85 | Ilias Tsitsimpis | run_burnin() { |
56 | e6ec7e85 | Ilias Tsitsimpis | local alias="$1" |
57 | e6ec7e85 | Ilias Tsitsimpis | local token="$2" |
58 | e6ec7e85 | Ilias Tsitsimpis | local image="$3" |
59 | e6ec7e85 | Ilias Tsitsimpis | local flavor="$4" |
60 | e6ec7e85 | Ilias Tsitsimpis | local success_subject="$TAG ($alias) $SUCCESS_SUBJECT" |
61 | e6ec7e85 | Ilias Tsitsimpis | local failure_subject="$TAG ($alias) $FAILURE_SUBJECT" |
62 | e6ec7e85 | Ilias Tsitsimpis | local logfolder="$LOGFOLDER/$alias" |
63 | e6ec7e85 | Ilias Tsitsimpis | local outputfile="$OUTPUTFOLDER/burnin-$alias.out" |
64 | e6ec7e85 | Ilias Tsitsimpis | local failed=false |
65 | e6ec7e85 | Ilias Tsitsimpis | local error_summary |
66 | e6ec7e85 | Ilias Tsitsimpis | local stale_subject |
67 | e6ec7e85 | Ilias Tsitsimpis | |
68 | e6ec7e85 | Ilias Tsitsimpis | # Save date-stamp to output |
69 | e6ec7e85 | Ilias Tsitsimpis | date > $outputfile |
70 | e6ec7e85 | Ilias Tsitsimpis | echo -e \ |
71 | e6ec7e85 | Ilias Tsitsimpis | "\n\n===== Burnin Output ========================================" \ |
72 | e6ec7e85 | Ilias Tsitsimpis | >> $outputfile |
73 | e6ec7e85 | Ilias Tsitsimpis | |
74 | e6ec7e85 | Ilias Tsitsimpis | # Check for stale servers/networks |
75 | e6ec7e85 | Ilias Tsitsimpis | $Burnin --token="$token" --auth-url="$AUTH_URL" --show-stale 2>&1 | \ |
76 | e6ec7e85 | Ilias Tsitsimpis | grep "test" >> $outputfile 2>&1 |
77 | e6ec7e85 | Ilias Tsitsimpis | if [ $? -ne 0 ]; then |
78 | e6ec7e85 | Ilias Tsitsimpis | # No stale servers/networks found. Run burnin |
79 | e6ec7e85 | Ilias Tsitsimpis | $Burnin --token="$token" \ |
80 | e6ec7e85 | Ilias Tsitsimpis | --action-timeout="$TIMEOUT" \ |
81 | e6ec7e85 | Ilias Tsitsimpis | --image-id="$image" \ |
82 | e6ec7e85 | Ilias Tsitsimpis | --log-folder="$logfolder" \ |
83 | e6ec7e85 | Ilias Tsitsimpis | --auth-url="$AUTH_URL" \ |
84 | e6ec7e85 | Ilias Tsitsimpis | --force-flavor="$flavor" \ |
85 | 33d1cf90 | Ilias Tsitsimpis | --system-images-user="$SYSTEM_IMAGES_USER" \ |
86 | e6ec7e85 | Ilias Tsitsimpis | --nofailfast \ |
87 | e6ec7e85 | Ilias Tsitsimpis | &>> $outputfile |
88 | e6ec7e85 | Ilias Tsitsimpis | |
89 | e6ec7e85 | Ilias Tsitsimpis | echo -e \ |
90 | e6ec7e85 | Ilias Tsitsimpis | "\n\n===== Burnin Logs ==========================================" \ |
91 | e6ec7e85 | Ilias Tsitsimpis | >> $outputfile |
92 | e6ec7e85 | Ilias Tsitsimpis | |
93 | e6ec7e85 | Ilias Tsitsimpis | # Search log files for errors |
94 | e6ec7e85 | Ilias Tsitsimpis | for file in `ls -1d $logfolder/* | tail -1`/*/detail* ; do |
95 | e6ec7e85 | Ilias Tsitsimpis | if egrep "(ERROR)|(FAILED)" $file > /dev/null; then |
96 | e6ec7e85 | Ilias Tsitsimpis | failed=true |
97 | e6ec7e85 | Ilias Tsitsimpis | echo "FILENAME: $file" >> $outputfile |
98 | e6ec7e85 | Ilias Tsitsimpis | echo "ERROR: " >> $outputfile |
99 | e6ec7e85 | Ilias Tsitsimpis | cat "$file" >> $outputfile |
100 | e6ec7e85 | Ilias Tsitsimpis | fi |
101 | e6ec7e85 | Ilias Tsitsimpis | done |
102 | e6ec7e85 | Ilias Tsitsimpis | |
103 | e6ec7e85 | Ilias Tsitsimpis | # Clean output file from escape characters |
104 | e6ec7e85 | Ilias Tsitsimpis | sed -ri "s/\x1B\[([0-9]{1,2}(;[0-9]{1,2})?)?[m|K]//g" $outputfile |
105 | e6ec7e85 | Ilias Tsitsimpis | |
106 | e6ec7e85 | Ilias Tsitsimpis | # Send emails |
107 | e6ec7e85 | Ilias Tsitsimpis | if $failed; then |
108 | e6ec7e85 | Ilias Tsitsimpis | error_summary="`cat $outputfile | \ |
109 | e6ec7e85 | Ilias Tsitsimpis | egrep "ERROR: test_|FAIL: test_" | \ |
110 | e6ec7e85 | Ilias Tsitsimpis | awk '{ print $2 }' | grep -v "^$" | \ |
111 | e6ec7e85 | Ilias Tsitsimpis | sed -e 's/_/ /g' | cut -d" " -f3- | \ |
112 | e6ec7e85 | Ilias Tsitsimpis | tr '\n' ',' | sed -e 's/,$//g' | \ |
113 | e6ec7e85 | Ilias Tsitsimpis | sed -e 's/,/, /g'`" |
114 | e6ec7e85 | Ilias Tsitsimpis | cat $outputfile | /usr/bin/mailx -E \ |
115 | e6ec7e85 | Ilias Tsitsimpis | -s "$failure_subject: $error_summary" $RECIPIENTS |
116 | e6ec7e85 | Ilias Tsitsimpis | # else |
117 | e6ec7e85 | Ilias Tsitsimpis | # cat $outputfile | /usr/bin/mailx -E \ |
118 | e6ec7e85 | Ilias Tsitsimpis | # -s "$success_subject" $RECIPIENTS |
119 | e6ec7e85 | Ilias Tsitsimpis | fi |
120 | e6ec7e85 | Ilias Tsitsimpis | else |
121 | e6ec7e85 | Ilias Tsitsimpis | # Burnin found stale servers/networks. Try to clean them |
122 | e6ec7e85 | Ilias Tsitsimpis | $Burnin --token="$token" --auth-url="$AUTH_URL" --delete-stale \ |
123 | e6ec7e85 | Ilias Tsitsimpis | >> $outputfile 2>&1 |
124 | e6ec7e85 | Ilias Tsitsimpis | if [ $? -ne 0 ]; then |
125 | e6ec7e85 | Ilias Tsitsimpis | stale_subject="$failure_subject: Couldn't delete stale servers/networks" |
126 | e6ec7e85 | Ilias Tsitsimpis | else |
127 | e6ec7e85 | Ilias Tsitsimpis | stale_subject="$success_subject: Stale servers/networks deleted" |
128 | e6ec7e85 | Ilias Tsitsimpis | fi |
129 | e6ec7e85 | Ilias Tsitsimpis | |
130 | e6ec7e85 | Ilias Tsitsimpis | # Clean output file from escape characters |
131 | e6ec7e85 | Ilias Tsitsimpis | sed -ri "s/\x1B\[([0-9]{1,2}(;[0-9]{1,2})?)?[m|K]//g" $outputfile |
132 | e6ec7e85 | Ilias Tsitsimpis | |
133 | e6ec7e85 | Ilias Tsitsimpis | # Send mail |
134 | e6ec7e85 | Ilias Tsitsimpis | cat $outputfile | /usr/bin/mailx -E \ |
135 | e6ec7e85 | Ilias Tsitsimpis | -s "$stale_subject" $RECIPIENTS |
136 | e6ec7e85 | Ilias Tsitsimpis | fi |
137 | e6ec7e85 | Ilias Tsitsimpis | } |
138 | e6ec7e85 | Ilias Tsitsimpis | |
139 | e6ec7e85 | Ilias Tsitsimpis | |
140 | e6ec7e85 | Ilias Tsitsimpis | # -------------------------------------------------------------------- |
141 | e6ec7e85 | Ilias Tsitsimpis | # For each user run burnin function |
142 | e6ec7e85 | Ilias Tsitsimpis | |
143 | e6ec7e85 | Ilias Tsitsimpis | ( |
144 | e6ec7e85 | Ilias Tsitsimpis | flock -xn 200 || exit 1 |
145 | e6ec7e85 | Ilias Tsitsimpis | |
146 | e6ec7e85 | Ilias Tsitsimpis | set ${USERS[@]} |
147 | e6ec7e85 | Ilias Tsitsimpis | |
148 | e6ec7e85 | Ilias Tsitsimpis | while [ -n "$1" ]; do |
149 | e6ec7e85 | Ilias Tsitsimpis | run_burnin "$1" "$2" "$3" "$4" & |
150 | e6ec7e85 | Ilias Tsitsimpis | shift 4 |
151 | e6ec7e85 | Ilias Tsitsimpis | done |
152 | e6ec7e85 | Ilias Tsitsimpis | |
153 | e6ec7e85 | Ilias Tsitsimpis | wait |
154 | e6ec7e85 | Ilias Tsitsimpis | ) 200>$LOCKFILE |