Revision 1b334175 tools/burnin
b/tools/burnin | ||
---|---|---|
41 | 41 |
USAGE = ("\tburnin -o OS_NAME [options...] instance_name ...") |
42 | 42 |
|
43 | 43 |
MAX_RETRIES = 3 |
44 |
LOG_HEADERS = { |
|
45 |
0: "- ", |
|
46 |
1: "* ", |
|
47 |
2: "" |
|
48 |
} |
|
44 | 49 |
|
45 | 50 |
class InstanceDown(Exception): |
46 | 51 |
"""The checked instance was not up""" |
... | ... | |
58 | 63 |
sys.exit(2) |
59 | 64 |
|
60 | 65 |
|
61 |
def Log(msg, indent=0):
|
|
66 |
def Log(msg, *args, **kwargs):
|
|
62 | 67 |
"""Simple function that prints out its argument. |
63 | 68 |
|
64 | 69 |
""" |
65 |
headers = { |
|
66 |
0: "- ", |
|
67 |
1: "* ", |
|
68 |
2: "" |
|
69 |
} |
|
70 |
if args: |
|
71 |
msg = msg % args |
|
72 |
indent = kwargs.get('indent', 0) |
|
70 | 73 |
sys.stdout.write("%*s%s%s\n" % (2*indent, "", |
71 |
headers.get(indent, " "), msg))
|
|
74 |
LOG_HEADERS.get(indent, " "), msg))
|
|
72 | 75 |
sys.stdout.flush() |
73 | 76 |
|
77 |
|
|
74 | 78 |
def Err(msg, exit_code=1): |
75 | 79 |
"""Simple error logging that prints to stderr. |
76 | 80 |
|
... | ... | |
290 | 294 |
try: |
291 | 295 |
val = fn(*args) |
292 | 296 |
if retry_count > 0 and retry_count < MAX_RETRIES: |
293 |
Log("Idempotent %s succeeded after %d retries" %
|
|
294 |
(msg, MAX_RETRIES - retry_count))
|
|
297 |
Log("Idempotent %s succeeded after %d retries",
|
|
298 |
msg, MAX_RETRIES - retry_count)
|
|
295 | 299 |
return val |
296 | 300 |
except Exception, err: # pylint: disable-msg=W0703 |
297 | 301 |
if retry_count == 0: |
298 |
Log("Non-idempotent %s failed, aborting" % (msg, ))
|
|
302 |
Log("Non-idempotent %s failed, aborting", msg)
|
|
299 | 303 |
raise |
300 | 304 |
elif retry_count == 1: |
301 |
Log("Idempotent %s repeated failure, aborting" % (msg, ))
|
|
305 |
Log("Idempotent %s repeated failure, aborting", msg)
|
|
302 | 306 |
raise |
303 | 307 |
else: |
304 |
Log("Idempotent %s failed, retry #%d/%d: %s" %
|
|
305 |
(msg, MAX_RETRIES - retry_count + 1, MAX_RETRIES, err))
|
|
308 |
Log("Idempotent %s failed, retry #%d/%d: %s",
|
|
309 |
msg, MAX_RETRIES - retry_count + 1, MAX_RETRIES, err)
|
|
306 | 310 |
self.MaybeRetry(retry_count - 1, msg, fn, *args) |
307 | 311 |
|
308 | 312 |
def _SetDebug(self, ops): |
... | ... | |
382 | 386 |
""" |
383 | 387 |
self.ClearFeedbackBuf() |
384 | 388 |
job_ids = [cli.SendJob(row[0], cl=self.cl) for row in jobs] |
385 |
Log("Submitted job ID(s) %s" % utils.CommaJoin(job_ids), indent=1)
|
|
389 |
Log("Submitted job ID(s) %s", utils.CommaJoin(job_ids), indent=1)
|
|
386 | 390 |
results = [] |
387 | 391 |
for jid, (_, iname) in zip(job_ids, jobs): |
388 |
Log("waiting for job %s for %s" % (jid, iname), indent=2)
|
|
392 |
Log("waiting for job %s for %s", jid, iname, indent=2)
|
|
389 | 393 |
try: |
390 | 394 |
results.append(cli.PollJob(jid, cl=self.cl, feedback_fn=self.Feedback)) |
391 | 395 |
except Exception, err: # pylint: disable-msg=W0703 |
392 |
Log("Job for %s failed: %s" % (iname, err))
|
|
396 |
Log("Job for %s failed: %s", iname, err)
|
|
393 | 397 |
if len(results) != len(jobs): |
394 | 398 |
raise BurninFailure() |
395 | 399 |
return results |
... | ... | |
494 | 498 |
|
495 | 499 |
Log("Creating instances") |
496 | 500 |
for pnode, snode, instance in mytor: |
497 |
Log("instance %s" % instance, indent=1)
|
|
501 |
Log("instance %s", instance, indent=1)
|
|
498 | 502 |
if self.opts.iallocator: |
499 | 503 |
pnode = snode = None |
500 | 504 |
msg = "with iallocator %s" % self.opts.iallocator |
... | ... | |
534 | 538 |
"""Grow both the os and the swap disks by the requested amount, if any.""" |
535 | 539 |
Log("Growing disks") |
536 | 540 |
for instance in self.instances: |
537 |
Log("instance %s" % instance, indent=1)
|
|
541 |
Log("instance %s", instance, indent=1)
|
|
538 | 542 |
for idx, growth in enumerate(self.disk_growth): |
539 | 543 |
if growth > 0: |
540 | 544 |
op = opcodes.OpGrowDisk(instance_name=instance, disk=idx, |
541 | 545 |
amount=growth, wait_for_sync=True) |
542 |
Log("increase disk/%s by %s MB" % (idx, growth), indent=2)
|
|
546 |
Log("increase disk/%s by %s MB", idx, growth, indent=2)
|
|
543 | 547 |
self.ExecOrQueue(instance, op) |
544 | 548 |
|
545 | 549 |
@_DoBatch(True) |
... | ... | |
547 | 551 |
"""Replace disks on primary and secondary for drbd8.""" |
548 | 552 |
Log("Replacing disks on the same nodes") |
549 | 553 |
for instance in self.instances: |
550 |
Log("instance %s" % instance, indent=1)
|
|
554 |
Log("instance %s", instance, indent=1)
|
|
551 | 555 |
ops = [] |
552 | 556 |
for mode in constants.REPLACE_DISK_SEC, constants.REPLACE_DISK_PRI: |
553 | 557 |
op = opcodes.OpReplaceDisks(instance_name=instance, |
554 | 558 |
mode=mode, |
555 | 559 |
disks=[i for i in range(self.disk_count)], |
556 | 560 |
early_release=self.opts.early_release) |
557 |
Log("run %s" % mode, indent=2)
|
|
561 |
Log("run %s", mode, indent=2)
|
|
558 | 562 |
ops.append(op) |
559 | 563 |
self.ExecOrQueue(instance, *ops) # pylint: disable-msg=W0142 |
560 | 564 |
|
... | ... | |
567 | 571 |
mytor = izip(islice(cycle(self.nodes), 2, None), |
568 | 572 |
self.instances) |
569 | 573 |
for tnode, instance in mytor: |
570 |
Log("instance %s" % instance, indent=1)
|
|
574 |
Log("instance %s", instance, indent=1)
|
|
571 | 575 |
if self.opts.iallocator: |
572 | 576 |
tnode = None |
573 | 577 |
msg = "with iallocator %s" % self.opts.iallocator |
... | ... | |
579 | 583 |
iallocator=self.opts.iallocator, |
580 | 584 |
disks=[], |
581 | 585 |
early_release=self.opts.early_release) |
582 |
Log("run %s %s" % (mode, msg), indent=2)
|
|
586 |
Log("run %s %s", mode, msg, indent=2)
|
|
583 | 587 |
self.ExecOrQueue(instance, op) |
584 | 588 |
|
585 | 589 |
@_DoCheckInstances |
... | ... | |
588 | 592 |
"""Failover the instances.""" |
589 | 593 |
Log("Failing over instances") |
590 | 594 |
for instance in self.instances: |
591 |
Log("instance %s" % instance, indent=1)
|
|
595 |
Log("instance %s", instance, indent=1)
|
|
592 | 596 |
op = opcodes.OpFailoverInstance(instance_name=instance, |
593 | 597 |
ignore_consistency=False) |
594 | 598 |
self.ExecOrQueue(instance, op) |
... | ... | |
601 | 605 |
mytor = izip(islice(cycle(self.nodes), 1, None), |
602 | 606 |
self.instances) |
603 | 607 |
for tnode, instance in mytor: |
604 |
Log("instance %s" % instance, indent=1)
|
|
608 |
Log("instance %s", instance, indent=1)
|
|
605 | 609 |
op = opcodes.OpMoveInstance(instance_name=instance, |
606 | 610 |
target_node=tnode) |
607 | 611 |
self.ExecOrQueue(instance, op) |
... | ... | |
611 | 615 |
"""Migrate the instances.""" |
612 | 616 |
Log("Migrating instances") |
613 | 617 |
for instance in self.instances: |
614 |
Log("instance %s" % instance, indent=1)
|
|
618 |
Log("instance %s", instance, indent=1)
|
|
615 | 619 |
op1 = opcodes.OpMigrateInstance(instance_name=instance, live=True, |
616 | 620 |
cleanup=False) |
617 | 621 |
|
... | ... | |
633 | 637 |
self.instances) |
634 | 638 |
|
635 | 639 |
for pnode, snode, enode, instance in mytor: |
636 |
Log("instance %s" % instance, indent=1)
|
|
640 |
Log("instance %s", instance, indent=1)
|
|
637 | 641 |
# read the full name of the instance |
638 | 642 |
nam_op = opcodes.OpQueryInstances(output_fields=["name"], |
639 | 643 |
names=[instance], use_locking=True) |
... | ... | |
681 | 685 |
|
682 | 686 |
erem_op = opcodes.OpRemoveExport(instance_name=instance) |
683 | 687 |
|
684 |
Log("export to node %s" % enode, indent=2)
|
|
688 |
Log("export to node %s", enode, indent=2)
|
|
685 | 689 |
Log("remove instance", indent=2) |
686 | 690 |
Log(import_log_msg, indent=2) |
687 | 691 |
Log("remove export", indent=2) |
... | ... | |
709 | 713 |
"""Stop/start the instances.""" |
710 | 714 |
Log("Stopping and starting instances") |
711 | 715 |
for instance in self.instances: |
712 |
Log("instance %s" % instance, indent=1)
|
|
716 |
Log("instance %s", instance, indent=1)
|
|
713 | 717 |
op1 = self.StopInstanceOp(instance) |
714 | 718 |
op2 = self.StartInstanceOp(instance) |
715 | 719 |
self.ExecOrQueue(instance, op1, op2) |
... | ... | |
719 | 723 |
"""Remove the instances.""" |
720 | 724 |
Log("Removing instances") |
721 | 725 |
for instance in self.to_rem: |
722 |
Log("instance %s" % instance, indent=1)
|
|
726 |
Log("instance %s", instance, indent=1)
|
|
723 | 727 |
op = opcodes.OpRemoveInstance(instance_name=instance, |
724 | 728 |
ignore_failures=True) |
725 | 729 |
self.ExecOrQueue(instance, op) |
... | ... | |
734 | 738 |
Log("Renaming instances") |
735 | 739 |
rename = self.opts.rename |
736 | 740 |
for instance in self.instances: |
737 |
Log("instance %s" % instance, indent=1)
|
|
741 |
Log("instance %s", instance, indent=1)
|
|
738 | 742 |
op_stop1 = self.StopInstanceOp(instance) |
739 | 743 |
op_stop2 = self.StopInstanceOp(rename) |
740 | 744 |
op_rename1 = self.RenameInstanceOp(instance, rename) |
... | ... | |
752 | 756 |
"""Reinstall the instances.""" |
753 | 757 |
Log("Reinstalling instances") |
754 | 758 |
for instance in self.instances: |
755 |
Log("instance %s" % instance, indent=1)
|
|
759 |
Log("instance %s", instance, indent=1)
|
|
756 | 760 |
op1 = self.StopInstanceOp(instance) |
757 | 761 |
op2 = opcodes.OpReinstallInstance(instance_name=instance) |
758 | 762 |
Log("reinstall without passing the OS", indent=2) |
... | ... | |
768 | 772 |
"""Reboot the instances.""" |
769 | 773 |
Log("Rebooting instances") |
770 | 774 |
for instance in self.instances: |
771 |
Log("instance %s" % instance, indent=1)
|
|
775 |
Log("instance %s", instance, indent=1)
|
|
772 | 776 |
ops = [] |
773 | 777 |
for reboot_type in constants.REBOOT_TYPES: |
774 | 778 |
op = opcodes.OpRebootInstance(instance_name=instance, |
775 | 779 |
reboot_type=reboot_type, |
776 | 780 |
ignore_secondaries=False) |
777 |
Log("reboot with type '%s'" % reboot_type, indent=2)
|
|
781 |
Log("reboot with type '%s'", reboot_type, indent=2)
|
|
778 | 782 |
ops.append(op) |
779 | 783 |
self.ExecOrQueue(instance, *ops) # pylint: disable-msg=W0142 |
780 | 784 |
|
... | ... | |
784 | 788 |
"""Activate and deactivate disks of the instances.""" |
785 | 789 |
Log("Activating/deactivating disks") |
786 | 790 |
for instance in self.instances: |
787 |
Log("instance %s" % instance, indent=1)
|
|
791 |
Log("instance %s", instance, indent=1)
|
|
788 | 792 |
op_start = self.StartInstanceOp(instance) |
789 | 793 |
op_act = opcodes.OpActivateInstanceDisks(instance_name=instance) |
790 | 794 |
op_deact = opcodes.OpDeactivateInstanceDisks(instance_name=instance) |
... | ... | |
800 | 804 |
"""Add and remove an extra disk for the instances.""" |
801 | 805 |
Log("Adding and removing disks") |
802 | 806 |
for instance in self.instances: |
803 |
Log("instance %s" % instance, indent=1)
|
|
807 |
Log("instance %s", instance, indent=1)
|
|
804 | 808 |
op_add = opcodes.OpSetInstanceParams(\ |
805 | 809 |
instance_name=instance, |
806 | 810 |
disks=[(constants.DDM_ADD, {"size": self.disk_size[0]})]) |
... | ... | |
817 | 821 |
"""Add and remove an extra NIC for the instances.""" |
818 | 822 |
Log("Adding and removing NICs") |
819 | 823 |
for instance in self.instances: |
820 |
Log("instance %s" % instance, indent=1)
|
|
824 |
Log("instance %s", instance, indent=1)
|
|
821 | 825 |
op_add = opcodes.OpSetInstanceParams(\ |
822 | 826 |
instance_name=instance, nics=[(constants.DDM_ADD, {})]) |
823 | 827 |
op_rem = opcodes.OpSetInstanceParams(\ |
... | ... | |
932 | 936 |
except Exception, err: # pylint: disable-msg=W0703 |
933 | 937 |
if has_err: # already detected errors, so errors in removal |
934 | 938 |
# are quite expected |
935 |
Log("Note: error detected during instance remove: %s" % str(err))
|
|
939 |
Log("Note: error detected during instance remove: %s", err)
|
|
936 | 940 |
else: # non-expected error |
937 | 941 |
raise |
938 | 942 |
|
Also available in: Unified diff