Revision 99d39120 ci/utils.py
| b/ci/utils.py | ||
|---|---|---|
| 1 |
#!/usr/bin/env python |
|
| 1 |
# !/usr/bin/env python
|
|
| 2 | 2 |
|
| 3 | 3 |
""" |
| 4 | 4 |
Synnefo ci utils module |
| ... | ... | |
| 155 | 155 |
self.temp_config.optionxform = str |
| 156 | 156 |
self.temp_config.read(self.temp_config_file) |
| 157 | 157 |
self.build_id = build_id |
| 158 |
self.logger.info("Will use \"%s\" as build id" % _green(self.build_id))
|
|
| 158 |
if build_id is not None: |
|
| 159 |
self.logger.info("Will use \"%s\" as build id" %
|
|
| 160 |
_green(self.build_id)) |
|
| 159 | 161 |
|
| 160 | 162 |
# Set kamaki cloud |
| 161 | 163 |
if cloud is not None: |
| ... | ... | |
| 225 | 227 |
self.logger.error( |
| 226 | 228 |
"Waiting for server to become %s timed out" % new_status) |
| 227 | 229 |
self.destroy_server(False) |
| 228 |
sys.exit(-1)
|
|
| 230 |
sys.exit(1) |
|
| 229 | 231 |
elif server['status'] == current_status: |
| 230 | 232 |
# Sleep for #n secs and continue |
| 231 | 233 |
timeout = timeout - sleep_time |
| ... | ... | |
| 234 | 236 |
self.logger.error( |
| 235 | 237 |
"Server failed with status %s" % server['status']) |
| 236 | 238 |
self.destroy_server(False) |
| 237 |
sys.exit(-1)
|
|
| 239 |
sys.exit(1) |
|
| 238 | 240 |
|
| 239 | 241 |
@_check_kamaki |
| 240 | 242 |
def destroy_server(self, wait=True): |
| ... | ... | |
| 562 | 564 |
self.logger.debug("Remote file has sha256 hash %s" % hash2)
|
| 563 | 565 |
if hash1 != hash2: |
| 564 | 566 |
self.logger.error("Hashes differ.. aborting")
|
| 565 |
sys.exit(-1)
|
|
| 567 |
sys.exit(1) |
|
| 566 | 568 |
|
| 567 | 569 |
@_check_fabric |
| 568 | 570 |
def clone_repo(self, local_repo=False): |
| ... | ... | |
| 577 | 579 |
self.config.get('Global', 'git_config_mail'))
|
| 578 | 580 |
_run(cmd, False) |
| 579 | 581 |
|
| 582 |
# Clone synnefo_repo |
|
| 583 |
synnefo_branch = self.clone_synnefo_repo(local_repo=local_repo) |
|
| 584 |
# Clone pithos-web-client |
|
| 585 |
self.clone_pithos_webclient_repo(synnefo_branch) |
|
| 586 |
|
|
| 587 |
@_check_fabric |
|
| 588 |
def clone_synnefo_repo(self, local_repo=False): |
|
| 589 |
"""Clone Synnefo repo to remote server""" |
|
| 580 | 590 |
# Find synnefo_repo and synnefo_branch to use |
| 581 | 591 |
synnefo_repo = self.config.get('Global', 'synnefo_repo')
|
| 582 | 592 |
synnefo_branch = self.config.get("Global", "synnefo_branch")
|
| ... | ... | |
| 590 | 600 |
subprocess.Popen( |
| 591 | 601 |
["git", "rev-parse", "--short", "HEAD"], |
| 592 | 602 |
stdout=subprocess.PIPE).communicate()[0].strip() |
| 593 |
self.logger.info("Will use branch \"%s\"" % _green(synnefo_branch))
|
|
| 603 |
self.logger.debug("Will use branch \"%s\"" % _green(synnefo_branch))
|
|
| 594 | 604 |
|
| 595 |
if local_repo or synnefo_branch == "":
|
|
| 605 |
if local_repo or synnefo_repo == "":
|
|
| 596 | 606 |
# Use local_repo |
| 597 | 607 |
self.logger.debug("Push local repo to server")
|
| 598 | 608 |
# Firstly create the remote repo |
| ... | ... | |
| 609 | 619 |
-q "$@"' > {4}
|
| 610 | 620 |
chmod u+x {4}
|
| 611 | 621 |
export GIT_SSH="{4}"
|
| 612 |
echo "{0}" | git push --mirror ssh://{1}@{2}:{3}/~/synnefo
|
|
| 622 |
echo "{0}" | git push --quiet --mirror ssh://{1}@{2}:{3}/~/synnefo
|
|
| 613 | 623 |
rm -f {4}
|
| 614 | 624 |
""".format(fabric.env.password, |
| 615 | 625 |
fabric.env.user, |
| ... | ... | |
| 619 | 629 |
os.system(cmd) |
| 620 | 630 |
else: |
| 621 | 631 |
# Clone Synnefo from remote repo |
| 622 |
# Currently clonning synnefo can fail unexpectedly |
|
| 623 |
cloned = False |
|
| 624 |
for i in range(10): |
|
| 625 |
self.logger.debug("Clone synnefo from %s" % synnefo_repo)
|
|
| 626 |
try: |
|
| 627 |
_run("git clone %s synnefo" % synnefo_repo, False)
|
|
| 628 |
cloned = True |
|
| 629 |
break |
|
| 630 |
except BaseException: |
|
| 631 |
self.logger.warning( |
|
| 632 |
"Clonning synnefo failed.. retrying %s" % i) |
|
| 633 |
if not cloned: |
|
| 634 |
self.logger.error("Can not clone Synnefo repo.")
|
|
| 635 |
sys.exit(-1) |
|
| 632 |
self.logger.debug("Clone synnefo from %s" % synnefo_repo)
|
|
| 633 |
self._git_clone(synnefo_repo) |
|
| 636 | 634 |
|
| 637 | 635 |
# Checkout the desired synnefo_branch |
| 638 | 636 |
self.logger.debug("Checkout \"%s\" branch/commit" % synnefo_branch)
|
| 639 | 637 |
cmd = """ |
| 640 | 638 |
cd synnefo |
| 641 |
for branch in `git branch -a | grep remotes | \ |
|
| 642 |
grep -v HEAD | grep -v master`; do |
|
| 639 |
for branch in `git branch -a | grep remotes | grep -v HEAD`; do |
|
| 643 | 640 |
git branch --track ${branch##*/} $branch
|
| 644 | 641 |
done |
| 645 | 642 |
git checkout %s |
| 646 | 643 |
""" % (synnefo_branch) |
| 647 | 644 |
_run(cmd, False) |
| 648 | 645 |
|
| 646 |
return synnefo_branch |
|
| 647 |
|
|
| 649 | 648 |
@_check_fabric |
| 650 |
def build_synnefo(self): |
|
| 651 |
"""Build Synnefo packages""" |
|
| 652 |
self.logger.info("Build Synnefo packages..")
|
|
| 653 |
self.logger.debug("Install development packages")
|
|
| 649 |
def clone_pithos_webclient_repo(self, synnefo_branch): |
|
| 650 |
"""Clone Pithos WebClient repo to remote server""" |
|
| 651 |
# Find pithos_webclient_repo and pithos_webclient_branch to use |
|
| 652 |
pithos_webclient_repo = \ |
|
| 653 |
self.config.get('Global', 'pithos_webclient_repo')
|
|
| 654 |
pithos_webclient_branch = \ |
|
| 655 |
self.config.get('Global', 'pithos_webclient_branch')
|
|
| 656 |
|
|
| 657 |
# Clone pithos-webclient from remote repo |
|
| 658 |
self.logger.debug("Clone pithos-webclient from %s" %
|
|
| 659 |
pithos_webclient_repo) |
|
| 660 |
self._git_clone(pithos_webclient_repo) |
|
| 661 |
|
|
| 662 |
# Track all pithos-webclient branches |
|
| 663 |
cmd = """ |
|
| 664 |
cd pithos-web-client |
|
| 665 |
for branch in `git branch -a | grep remotes | grep -v HEAD`; do |
|
| 666 |
git branch --track ${branch##*/} $branch > /dev/null 2>&1
|
|
| 667 |
done |
|
| 668 |
git branch |
|
| 669 |
""" |
|
| 670 |
webclient_branches = _run(cmd, False) |
|
| 671 |
webclient_branches = webclient_branches.split() |
|
| 672 |
|
|
| 673 |
# If we have pithos_webclient_branch in config file use this one |
|
| 674 |
# else try to use the same branch as synnefo_branch |
|
| 675 |
# else use an appropriate one. |
|
| 676 |
if pithos_webclient_branch == "": |
|
| 677 |
if synnefo_branch in webclient_branches: |
|
| 678 |
pithos_webclient_branch = synnefo_branch |
|
| 679 |
else: |
|
| 680 |
# If synnefo_branch starts with one of |
|
| 681 |
# 'master', 'hotfix'; use the master branch |
|
| 682 |
if synnefo_branch.startswith('master') or \
|
|
| 683 |
synnefo_branch.startswith('hotfix'):
|
|
| 684 |
pithos_webclient_branch = "master" |
|
| 685 |
# If synnefo_branch starts with one of |
|
| 686 |
# 'develop', 'feature'; use the develop branch |
|
| 687 |
elif synnefo_branch.startswith('develop') or \
|
|
| 688 |
synnefo_branch.startswith('feature'):
|
|
| 689 |
pithos_webclient_branch = "develop" |
|
| 690 |
else: |
|
| 691 |
self.logger.waring( |
|
| 692 |
"Cannot determine which pithos-web-client branch to " |
|
| 693 |
"use based on \"%s\" synnefo branch. " |
|
| 694 |
"Will use develop." % synnefo_branch) |
|
| 695 |
pithos_webclient_branch = "develop" |
|
| 696 |
# Checkout branch |
|
| 697 |
self.logger.debug("Checkout \"%s\" branch" %
|
|
| 698 |
_green(pithos_webclient_branch)) |
|
| 699 |
cmd = """ |
|
| 700 |
cd pithos-web-client |
|
| 701 |
git checkout {0}
|
|
| 702 |
""".format(pithos_webclient_branch) |
|
| 703 |
_run(cmd, False) |
|
| 704 |
|
|
| 705 |
def _git_clone(self, repo): |
|
| 706 |
"""Clone repo to remote server |
|
| 707 |
|
|
| 708 |
Currently clonning from code.grnet.gr can fail unexpectedly. |
|
| 709 |
So retry!! |
|
| 710 |
|
|
| 711 |
""" |
|
| 712 |
cloned = False |
|
| 713 |
for i in range(1, 11): |
|
| 714 |
try: |
|
| 715 |
_run("git clone %s" % repo, False)
|
|
| 716 |
cloned = True |
|
| 717 |
break |
|
| 718 |
except BaseException: |
|
| 719 |
self.logger.warning("Clonning failed.. retrying %s/10" % i)
|
|
| 720 |
if not cloned: |
|
| 721 |
self.logger.error("Can not clone repo.")
|
|
| 722 |
sys.exit(1) |
|
| 723 |
|
|
| 724 |
@_check_fabric |
|
| 725 |
def build_packages(self): |
|
| 726 |
"""Build packages needed by Synnefo software""" |
|
| 727 |
self.logger.info("Install development packages")
|
|
| 654 | 728 |
cmd = """ |
| 655 | 729 |
apt-get update |
| 656 | 730 |
apt-get install zlib1g-dev dpkg-dev debhelper git-buildpackage \ |
| 657 |
python-dev python-all python-pip --yes --force-yes |
|
| 731 |
python-dev python-all python-pip ant --yes --force-yes
|
|
| 658 | 732 |
pip install -U devflow |
| 659 | 733 |
""" |
| 660 | 734 |
_run(cmd, False) |
| 661 | 735 |
|
| 736 |
# Patch pydist bug |
|
| 662 | 737 |
if self.config.get('Global', 'patch_pydist') == "True":
|
| 663 | 738 |
self.logger.debug("Patch pydist.py module")
|
| 664 | 739 |
cmd = r""" |
| ... | ... | |
| 668 | 743 |
_run(cmd, False) |
| 669 | 744 |
|
| 670 | 745 |
# Build synnefo packages |
| 671 |
self.logger.debug("Build synnefo packages")
|
|
| 746 |
self.build_synnefo() |
|
| 747 |
# Build pithos-web-client packages |
|
| 748 |
self.build_pithos_webclient() |
|
| 749 |
|
|
| 750 |
@_check_fabric |
|
| 751 |
def build_synnefo(self): |
|
| 752 |
"""Build Synnefo packages""" |
|
| 753 |
self.logger.info("Build Synnefo packages..")
|
|
| 754 |
|
|
| 672 | 755 |
cmd = """ |
| 673 | 756 |
devflow-autopkg snapshot -b ~/synnefo_build-area --no-sign |
| 674 | 757 |
""" |
| ... | ... | |
| 693 | 776 |
_run(cmd, False) |
| 694 | 777 |
|
| 695 | 778 |
@_check_fabric |
| 779 |
def build_pithos_webclient(self): |
|
| 780 |
"""Build pithos-web-client packages""" |
|
| 781 |
self.logger.info("Build pithos-web-client packages..")
|
|
| 782 |
|
|
| 783 |
cmd = """ |
|
| 784 |
devflow-autopkg snapshot -b ~/webclient_build-area --no-sign |
|
| 785 |
""" |
|
| 786 |
with fabric.cd("pithos-web-client"):
|
|
| 787 |
_run(cmd, True) |
|
| 788 |
|
|
| 789 |
# Setup pithos-web-client packages for snf-deploy |
|
| 790 |
self.logger.debug("Copy webclient debs to snf-deploy packages dir")
|
|
| 791 |
cmd = """ |
|
| 792 |
cp ~/webclient_build-area/*.deb /var/lib/snf-deploy/packages/ |
|
| 793 |
""" |
|
| 794 |
_run(cmd, False) |
|
| 795 |
|
|
| 796 |
@_check_fabric |
|
| 696 | 797 |
def build_documentation(self): |
| 697 | 798 |
"""Build Synnefo documentation""" |
| 698 | 799 |
self.logger.info("Build Synnefo documentation..")
|
| ... | ... | |
| 824 | 925 |
if not os.path.exists(dest): |
| 825 | 926 |
os.makedirs(dest) |
| 826 | 927 |
self.fetch_compressed("synnefo_build-area", dest)
|
| 928 |
self.fetch_compressed("webclient_build-area", dest)
|
|
| 827 | 929 |
self.logger.info("Downloaded debian packages to %s" %
|
| 828 | 930 |
_green(dest)) |
| 829 | 931 |
|
Also available in: Unified diff