Synnefo

#!/bin/bash

REMOTEUPSTREAM=develop

REMOTEDEBIAN=debian-develop

PKGAREA=~/packages

BACKUPAREA=~/backup

BUILDAREA=$(mktemp -d --tmpdir=/tmp build-area.XXX)

PACKAGES="

  snf-quotaholder-app

  snf-astakos-app

  snf-common

  snf-webproject

  snf-cyclades-app

  snf-cyclades-gtools

  snf-tools

  snf-pithos-app

  snf-pithos-backend

  snf-pithos-tools"

# Copyright 2012 GRNET S.A. All rights reserved.

#

# Redistribution and use in source and binary forms, with or

# without modification, are permitted provided that the following

# conditions are met:

#

#   1. Redistributions of source code must retain the above

#      copyright notice, this list of conditions and the following

#      disclaimer.

#

#   2. Redistributions in binary form must reproduce the above

#      copyright notice, this list of conditions and the following

#      disclaimer in the documentation and/or other materials

#      provided with the distribution.

#

# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS

# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED

# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR

# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR

# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,

# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT

# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF

# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED

# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT

# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN

# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE

# POSSIBILITY OF SUCH DAMAGE.

#

# The views and conclusions contained in the software and

# documentation are those of the authors and should not be

# interpreted as representing official policies, either expressed

# or implied, of GRNET S.A.

import git

import os

import sys

from sh import mktemp, cd, rm, git_dch, python

from optparse import OptionParser

try:

    from colors import red, green

except ImportError:

    red = lambda x: x

    green = lambda x: x

print_red = lambda x: sys.stdout.write(red(x) + "\n")

print_green = lambda x: sys.stdout.write(green(x) + "\n")

AVAILABLE_MODES = ["release", "snapshot"]

PACKAGES = (

  "snf-astakos-app",

  "snf-common",

  "snf-webproject",

  "snf-cyclades-app",

  "snf-cyclades-gtools",

  "snf-tools",

  "snf-pithos-app",

  "snf-pithos-backend",

  "snf-pithos-tools",

)

def main():

    parser = OptionParser(usage="usage: %prog [options] mode",

                          version="%prog 1.0")

    parser.add_option("-k", "--keep-repo",

                      action="store_true",

                      dest="keep_repo",

                      default=False,

                      help="Do not delete the cloned repository")

    parser.add_option("-b", "--build-dir",

                      dest="build_dir",

                      default=None,

                      help="Directory to store created pacakges")

    parser.add_option("-r", "--repo-dir",

                     dest="repo_dir",

                     default=None,

                     help="Directory to clone repository")

    parser.add_option("-d", "--dirty",

                     dest="force_dirty",

                     default=False,

                     action="store_true",

                     help="Do not check if working directory is dirty")

    (options, args) = parser.parse_args()

    mode = args[0]

    if mode not in AVAILABLE_MODES:

        raise ValueError(red("Invalid argument! Mode must be one: %s"

                         % ", ".join(AVAILABLE_MODES)))

    # Do not prompt for merge message. Required for some Git versions

    os.environ["GITFLOW_BUILD_MODE"] = mode

    try:

        original_repo = git.Repo(".")

    except git.git.InvalidGitRepositoryError:

        raise RuntimeError(red("Current directory is not git repository."))

    if original_repo.is_dirty() and not options.force_dirty:

        toplevel = original_repo.working_dir

        raise RuntimeError(red("Repository %s is dirty." % toplevel))

    repo_dir = options.repo_dir

    if not repo_dir:

        repo_dir = mktemp("-d", "/tmp/synnefo-build-repo-XXX").stdout.strip()

        print_green("Created temporary directory '%s' for the cloned repo."

                    % repo_dir)

    repo = original_repo.clone(repo_dir)

    print_green("Cloned current repository to '%s'." % repo_dir)

    reflog_hexsha = repo.head.log()[-1].newhexsha

    print "Latest Reflog entry is %s" % reflog_hexsha

    branch = repo.head.reference.name

    if branch == "master":

        debian_branch = "debian"

    else:

        debian_branch = "debian-" + branch

    try:

        repo.references[debian_branch]

    except IndexError:

        # Branch does not exist

        # FIXME: remove hard-coded strings..

        if branch == "debian":

            repo.git.branch("--track", debian_branch, "origin/debian")

        else:

            repo.git.branch("--track", debian_branch, "origin/debian-develop")

    repo.git.checkout(debian_branch)

    print_green("Changed to branch '%s'" % debian_branch)

    repo.git.merge(branch)

    print_green("Merged branch '%s' into '%s'" % (branch, debian_branch))

    cd(repo_dir)

    version = python(repo_dir + "/devtools/version.py", "debian").strip()

    print_green("The new debian version will be: '%s'" % version)

    dch = git_dch("--debian-branch=%s" % debian_branch,

            "--git-author",

            "--ignore-regex=\".*\"",

            "--multimaint-merge",

            "--since=HEAD",

            "--new-version=%s" % version)

    print_green("Successfully ran '%s'" % " ".join(dch.cmd))

    os.system("vim debian/changelog")

    repo.git.add("debian/changelog")

    if mode == "release":

        repo.git.commit("-s", "-a", "-m", "Bump new upstream version")

        if branch == "master":

            repo.git.tag("debian/" + version)

    for package in PACKAGES:

        # python setup.py should run in its directory

        cd(package)

        package_dir = repo_dir + "/" + package

        res = python(package_dir + "/setup.py", "sdist", _out=sys.stdout)

        cd("../")

        print res.stdout

    # Add version.py files to repo

    os.system("grep \"__version_vcs\" -r . -l -I | xargs git add -f")

    build_dir = options.build_dir

    if not options.build_dir:

        build_dir = mktemp("-d", "/tmp/synnefo-build-XXX").stdout.strip()

        print_green("Created directory '%s' to store the .deb files." %

                     build_dir)

    os.system("git-buildpackage --git-export-dir=%s --git-upstream-branch=%s"

              " --git-debian-branch=%s --git-export=INDEX --git-ignore-new -sa"

              % (build_dir, branch, debian_branch))

    if not options.keep_repo:

        print_green("Removing cloned repo '%s'." % repo_dir)

        rm("-r", repo_dir)

    else:

        print_green("Repository dir '%s'" % repo_dir)

    print_green("Completed. Version '%s', build area: '%s'"

                % (version, build_dir))

if __name__ == "__main__":

    main()

#!/bin/bash

# Copyright 2012 GRNET S.A. All rights reserved.

# 

# Redistribution and use in source and binary forms, with or

# without modification, are permitted provided that the following

# conditions are met:

# 

#   1. Redistributions of source code must retain the above

#      copyright notice, this list of conditions and the following

#      disclaimer.

# 

#   2. Redistributions in binary form must reproduce the above

#      copyright notice, this list of conditions and the following

#      disclaimer in the documentation and/or other materials

#      provided with the distribution.

# 

# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS

# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED

# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR

# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR

# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,

# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT

# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF

# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED

# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT

# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN

# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE

# POSSIBILITY OF SUCH DAMAGE.

# 

# The views and conclusions contained in the software and

# documentation are those of the authors and should not be

# interpreted as representing official policies, either expressed

# or implied, of GRNET S.A.

parse_git_branch()

{

    git branch 2> /dev/null | grep '^*' | sed 's/^*\ //g'

}

die()

{

    echo $* 1>&2

    exit 1

}

cleanup()

{

    trap - EXIT

    if [ ${#CLEANUP[*]} -gt 0 ]; then

        LAST_ELEMENT=$((${#CLEANUP[*]}-1))

        REVERSE_INDEXES=$(seq ${LAST_ELEMENT} -1 0)

        for i in $REVERSE_INDEXES; do

            local cmd=${CLEANUP[$i]}

            $cmd

        done

    fi

}

add_cleanup() {

    local cmd=""

    for arg; do cmd+=$(printf "%q " "$arg"); done

    CLEANUP+=("$cmd")

}

add_checkpoint()

{

    commit=$(git reflog | head -n1 | cut -f 1 -d " ")

    add_cleanup git reset --hard $commit

    LASTCHECKPOINT=$commit

}

CLEANUP=( )

source devtools/autopkg.conf

# The root of the git repository, no matter where we're called from

TOPLEVEL="$(git rev-parse --show-toplevel)"

CURRENT_BRANCH=$(parse_git_branch)

LOCALBRANCH="$CURRENT_BRANCH"

LOCALDEBIAN=$1

DEBIANBRANCH=${LOCALDEBIAN:- origin/$REMOTEDEBIAN}

MODIFIED=$(git status --short | grep -v "??")

if [[ -n $MODIFIED ]]; then

        echo "error: Repository is dirty. Commit your local changes."

        exit 1

fi

set -e

trap cleanup EXIT

cd "$TOPLEVEL"

# Prerequisites: Test all important directories exist

test -d "$PKGAREA" || die "Package area directory $PKGAREA missing"

test -d "$BACKUPAREA" || die "Backup area directory $BACKUPAREA missing"

# Prerequisite: Test the dialog utility is available

dialog --help &>/dev/null || die "Could not run the 'dialog' utility"

echo "##########################################################"

echo "Will build packages"

echo "under '$BUILDAREA',"

echo "from local branch '$LOCALBRANCH'"

echo "and debian branch '$DEBIANBRANCH'"

echo "##########################################################"

echo "Press Enter to continue..."

read

add_checkpoint

# Create a temporary debian branch to do everything

TMPDEBIAN=$(mktemp -u debian.XXX)

git branch --track $TMPDEBIAN  $DEBIANBRANCH

#add_cleanup git branch -D $TMPDEBIAN

git checkout $TMPDEBIAN

add_cleanup git checkout $LOCALBRANCH

add_checkpoint

# Whether we are in snapshot or release mode

snap=false

mrgextra=-m

dchextra=-R

mrgmsg="Merge branch '$REMOTEUPSTREAM' into $REMOTEDEBIAN"

dialog --yesno "Create Snapshot?" 5 20 && snap=true && GITFLOW_BUILD_MODE=snapshot && dchextra=-S && mrgextra= && mrgmsg=

# merge local branch into tmp branch with a nice commit message,

# so it can be pushed as is to upstream debian

GIT_MERGE_AUTOEDIT=no

git merge $mrgextra ${mrgextra:+"$mrgmsg"} $LOCALBRANCH

# auto edit Debian changelog depending on Snapshot or Release mode

export EDITOR=/usr/bin/vim

# use the devtools to determine Debian version

export GITFLOW_BUILD_MODE

version=$(devtools/version.py debian)

git-dch --debian-branch=$TMPDEBIAN --git-author --ignore-regex=".*" --multimaint-merge --since=HEAD -N $version

git add debian/changelog

# get version from the changelog

# we add a git tag here, so setup.py sdist works as expected

# FIXME: This is a workaround for the way Synnefo packages determine

#        the versions for their Python packages

version=$(IFS="()" ; read  x v x < debian/changelog  ; echo $v)

if ! $snap; then

  git commit -s -a -m "Bump new upstream version"

  TAGFILE=$(mktemp -t tag.XXX)

  add_cleanup rm $TAGFILE

  dialog --inputbox "New Debian Tag: " 5 30 "debian/$version" 2>$TAGFILE

  git tag $(<$TAGFILE)

  add_cleanup git tag -d $(<$TAGFILE)

fi

for p in $PACKAGES; do

  cd  $p

  python setup.py sdist

  grep "__version_vcs" -r . -l -I | xargs git add -f

  cd -

done

# Build all packages

git-buildpackage --git-export-dir="$BUILDAREA" \

                 --git-upstream-branch=$LOCALBRANCH \

                 --git-debian-branch=$TMPDEBIAN \

                 --git-export=INDEX \

                 --git-ignore-new -sa

# do some dirty backup

# pkgarea might be needed by auto-deploy tool

rm -f "$PKGAREA"/* || true

cp -v "$BUILDAREA"/* "$PKGAREA"/ || true

cp -v "$BUILDAREA"/* "$BACKUPAREA"/ || true

echo "###############################################"

echo "####              SUCCESS                  ####"

echo "###############################################"

git fetch origin

#check if your local branch is up-to-date

commits_behind=$(git rev-list $LOCALBRANCH..origin/$REMOTEUPSTREAM | wc -l)

if [ $commits_behind -ne 0 ]; then

  die "Your local branch is outdated!! Please run: git pull --rebase origin/$REMOTEUPSTREAM"

fi

commits_behind=$(git rev-list $DEBIANBRANCH..origin/$REMOTEDEBIAN | wc -l)

if [ $commits_behind -ne 0 ]; then

  die "Your debian branch is outdated!! Please run: git pull --rebase origin/$REMOTEDEBIAN"

fi

trap - EXIT

# Remove the added versions.py files

git reset --hard HEAD

# here we can push the commits to the remote debian branch as they are

if ! $snap; then

  TAGS="--tags"

fi

echo "git push $TAGS origin $TMPDEBIAN:$REMOTEDEBIAN"

echo "git checkout $LOCALBRANCH"

echo "git push $TAGS origin $LOCALBRANCH:$REMOTEUPSTREAM"

echo

exit 0

#!/usr/bin/env python

#

# Copyright 2012 GRNET S.A. All rights reserved.

#

# Redistribution and use in source and binary forms, with or

# without modification, are permitted provided that the following

# conditions are met:

#

#   1. Redistributions of source code must retain the above

#      copyright notice, this list of conditions and the following

#      disclaimer.

#

#   2. Redistributions in binary form must reproduce the above

#      copyright notice, this list of conditions and the following

#      disclaimer in the documentation and/or other materials

#      provided with the distribution.

#

# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS

# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED

# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR

# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR

# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,

# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT

# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF

# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED

# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT

# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN

# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE

# POSSIBILITY OF SUCH DAMAGE.

#

# The views and conclusions contained in the software and

# documentation are those of the authors and should not be

# interpreted as representing official policies, either expressed

# or implied, of GRNET S.A.

#

#

"""Unit Tests for devtools.version

Provides unit tests for module devtools.version,

for automatic generation of version strings.

"""

import os

import unittest

from pkg_resources import parse_version

from version import debian_version_from_python_version

class DebianVersionObject(object):

    """Object representing a Debian Version."""

    def __init__(self, pyver):

        self.version = debian_version_from_python_version(pyver)

    def __str__(self):

        return self.version

def debian_compare_versions(a, op, b):

    i = os.system("dpkg --compare-versions %s %s %s" % (a, op, b))

    return i == 0

# Set ordering between DebianVersionObject objects, by adding

# debian_compare_versions

for op in ["lt", "le", "eq", "ne", "gt", "ge"]:

    def gen(op):

        def operator_func(self, other):

            return debian_compare_versions(self.version, op, other.version)

        return operator_func

    setattr(DebianVersionObject, "__%s__" % op, gen(op))

def _random_commit():

    import random

    import string

    return "".join(random.choice(string.hexdigits) for n in xrange(8)).lower()

# Add a random commit number at the end of snapshot versions

def version_with_commit(parse_func, v):

    if "_" in v:

        return parse_func(v + "_" + _random_commit())

    else:

        return parse_func(v)

V = lambda v: version_with_commit(parse_version, v)

D = lambda v: version_with_commit(DebianVersionObject, v)

class TestVersionFunctions(unittest.TestCase):

    def setUp(self):

        self.version_orderings = (

            ("0.14next", ">", "0.14"),

            ("0.14next", ">", "0.14rc7"),

            ("0.14next", "<", "0.14.1"),

            ("0.14rc6", "<", "0.14"),

            ("0.14.2rc6", ">", "0.14.1"),

            ("0.14next_150", "<", "0.14next"),

            ("0.14.1next_150", "<", "0.14.1next"),

            ("0.14.1_149", "<", "0.14.1"),

            ("0.14.1_149", "<", "0.14.1_150"),

            ("0.13next_102", "<", "0.13next"),

            ("0.13next", "<", "0.14rc5_120"),

            ("0.14rc3_120", "<", "0.14rc3"),

            # The following test fails, but version.python_version

            # will never try to produce such a version:

            # ("0.14rc3", "<", "0.14_1"),

            ("0.14_120", "<", "0.14"),

            ("0.14", "<", "0.14next_20"),

            ("0.14next_20", "<", "0.14next"),

        )

    def test_python_versions(self):

        for a, op, b in self.version_orderings:

            res = compare(V, a, op, b)

            self.assertTrue(res, "Python version: %s %s %s"

                                 " is not True" % (a, op, b))

    def test_debian_versions(self):

        for a, op, b in self.version_orderings:

            res = compare(D, a, op, b)

            self.assertTrue(res, "Debian version %s %s %s"

                                 " is not True" % (a, op, b))

def compare(function, a, op, b):

    import operator

    str_to_op = {"<": operator.lt,

            "<=": operator.le,

            "==": operator.eq,

            ">": operator.gt,

            ">=": operator.ge}

    try:

        return str_to_op[op](function(a), function(b))

    except KeyError:

        raise ValueError("Unknown operator '%s'" % op)

if __name__ == '__main__':

    unittest.main()

# Copyright 2011 GRNET S.A. All rights reserved.

#

# Redistribution and use in source and binary forms, with or

# without modification, are permitted provided that the following

# conditions are met:

#

#   1. Redistributions of source code must retain the above

#      copyright notice, this list of conditions and the following

#      disclaimer.

#

#   2. Redistributions in binary form must reproduce the above

#      copyright notice, this list of conditions and the following

#      disclaimer in the documentation and/or other materials

#      provided with the distribution.

#

# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS

# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED

# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR

# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR

# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,

# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT

# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF

# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED

# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT

# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN

# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE

# POSSIBILITY OF SUCH DAMAGE.

#

# The views and conclusions contained in the software and

# documentation are those of the authors and should not be

# interpreted as representing official policies, either expressed

# or implied, of GRNET S.A.

#

import os

import sys

from contextlib import contextmanager

from fabric.api import *

from fabric.colors import *

env.project_root = "./"

env.develop = False

env.autoremove = True

env.packages = ['snf-common', 'snf-cyclades-app', 'snf-cyclades-gtools',

                'snf-webproject', 'snf-pithos-backend', 'snf-pithos-app',

                'snf-pithos-tools', 'snf-astakos-app']

env.capture = False

env.colors = True

env.pypi_root = 'pypi'

env.roledefs = {

    'docs': ['docs.dev.grnet.gr'],

    'pypi': ['docs.dev.grnet.gr']

}

# colored logging

notice = lambda x: sys.stdout.write(yellow(x) + "\n")

info = lambda x: sys.stdout.write(green(x) + "\n")

error = lambda x: sys.stdout.write(red(x) + "\n")

def dev():

    env.develop = True

# wrap local to respect global capturing setting from env.capture

oldlocal = local

def local(cmd, capture="default"):

    if capture != "default":

        capture = capture

    else:

        capture = env.capture

    return oldlocal(cmd, capture=capture)

def package_root(p):

    return os.path.join(env.project_root, p)

def remove_pkg(p):

    notice("uninstalling package: %s" % p)

    with lcd(package_root(p)):

        with settings(warn_only=True):

            local("pip uninstall %s -y" % p, env.capture)

def build_pkg(p):

    info("building package: %s" % p)

    with lcd(package_root(p)):

        local("if [ -d dist ]; then rm -r dist; fi;")

        local("if [ -d build ]; then rm -r build; fi;")

        local("python setup.py egg_info -d sdist")

def install_pkg(p):

    info("installing package: %s" % p)

    with lcd(package_root(p)):

        if env.develop:

            local("python setup.py develop")

        else:

            local("python setup.py install")

def install(*packages):

    for p in packages:

        install_pkg("snf-%s" % p)

def buildall():

    for p in env.packages:

        build_pkg(p)

    collectdists()

def installall():

    for p in env.packages:

        install_pkg(p)

def collectdists():

    if os.path.exists("./packages"):

        notice("removing 'packages' directory")

        local("rm -r packages")

    local("mkdir packages")

    for p in env.packages:

        local("cp %s/dist/*.tar.gz ./packages/" % package_root(p))

def removeall():

    for p in env.packages:

        remove_pkg(p)

def remove(*packages):

    for p in packages:

        remove_pkg("snf-%s" % p)

#

# GIT helpers

#

def git(params, locl=True):

    cmd = local if locl else run

    return cmd("git %s" % params, capture=True)

def branch():

    return git("symbolic-ref HEAD").split("/")[-1]

@contextmanager

def co(c):

    current_branch = branch()

    git("checkout %s" % c)

    # Use a try block to make sure we checkout the original branch.

    try:

        yield

    finally:

        try:

            git("checkout %s" % current_branch)

        except Exception:

            error("Could not checkout %s, you're still left at %s" % c)

#

# Debian packaging helpers

#

env.debian_branch = 'debian'

env.deb_packages = ['snf-common', 'snf-cyclades-app', 'snf-cyclades-gtools',

                    'snf-webproject', 'snf-pithos-backend', 'snf-pithos-tools',

                    'snf-pithos-app', 'snf-astakos-app']

env.signdebs = True

env.debrelease = False  # Increase release number in Debian changelogs

def _last_commit(f):

    return local("git rev-list --all --date-order --max-count=1 %s" % f,

                 capture=True).strip()

def _diff_from_master(c,f):

    return local("git log --oneline %s..master %s" \

                 " | wc -l" % (c, f), capture=True)

def dch(p):

    with co(env.debian_branch):

        local("git merge master")

        local("git merge %s" % env.upstream)

        with lcd(package_root(p)):

            local("if [ ! -d .git ]; then mkdir .git; fi")

            # FIXME:

            # Checking for new changes in packages

            # has been removed temporarily.

            # Always create a new Debian changelog entry.

            ## Check for new changes in package dir

            #diff = _diff_from_master(_last_commit("debian/changelog"), ".")

            #vercmd  = "git describe --tags --abbrev=0"\

            #          " | sed -rn '\''s/^v(.*)/\\1/p'\''"

            #version = local(vercmd, capture=True)

            #if int(diff) > 0:

            if True:

                # Run git-dch in snapshot mode.

                # TODO: Support a --release mode in fabfile

                if not env.debrelease:

                    notice(("Producing snapshot changelog entry, "

                            "use 'debrelease' to produce release entries."))

                local(("git-dch --debian-branch=%s --auto %s" %

                       (env.debian_branch,

                        "--release" if env.debrelease else "--snapshot")))

                local(("git commit debian/changelog"

                       " -m 'Updated %s changelog'" % p))

                notice(("Make sure to tag Debian release in %s" %

                        env.debian_branch))

            local("rmdir .git")

def dchall():

    for p in env.deb_packages:

        info("updating debian changelog for package: %s" % p)

        dch(p)

def debrelease():

    env.debrelease = True

def signdebs():

    env.signdebs = True

# Commands which automatically add and reset the version files which are not tracked by

# git. Those version files are created from each setup.py using the synnefo-common

# update_version, so we execute `python setup.py clean` to ensure that file gets

# created and git add will not fail. The reset of those files after each build

# certifies that succeded git checkouts will not fail due to existing local

# changes.

add_versions_cmd = "find . -regextype posix-egrep -regex \".*version.py$|.*\/synnefo\/versions\/.*py$\" -exec git add -f {} \;"

reset_versions_cmd = "find . -regextype posix-egrep -regex \".*version.py$|.*\/synnefo\/versions\/.*py$\" -exec git reset {} \;"

def builddeb(p, master="master", branch="debian-0.8"):

    with co(branch):

        info("Building debian package for %s" % p)

        with lcd(package_root(p)):

            local("git merge master")

            local("if [ ! -d .git ]; then mkdir .git; fi")

            local("python setup.py clean")

            local(add_versions_cmd)

            local(("git-buildpackage --git-upstream-branch=%s --git-debian-branch=%s"

                   " --git-export=INDEX --git-ignore-new %s") %

                  (master, branch, "" if env.signdebs else "-us -uc"))

            local("rm -rf .git")

            local(reset_versions_cmd)

        info("Done building debian package for %s" % p)

def builddeball(b="debian-0.8"):

    for p in env.deb_packages:

        builddeb(p=p, branch=b)

@roles('pypi')

def uploadtars():

    put("packages/*.tar.gz", 'www/pypi/')

def cleandocs():

    """

    Remove _build directories for each doc project

    """

    # docs contains conf.py in root directory

    if os.path.exists("docs/docs/_build"):

        local("rm -r docs/docs/_build")

    for p in env.packages:

        buildpth = os.path.join(package_root(p), 'docs', '_build')

        if os.path.exists(buildpth):

            local('rm -r %s' % buildpth)

def builddocs():

    """

    Run sphinx builder for each project separately

    """

    builddocs_cmd = "sphinx-build -b html -d _build/doctrees   . _build/html"

    # docs contains conf.py in root directory

    with lcd("docs"):

        local(builddocs_cmd)

    for p in env.packages:

        info("Building %s docs" % p)

        docspth = os.path.join(package_root(p), 'docs')

        if os.path.exists(docspth):

            with lcd(docspth):

                local(builddocs_cmd)

#!/bin/bash

#

#

# Copyright 2011 GRNET S.A. All rights reserved.

#

# Redistribution and use in source and binary forms, with or

# without modification, are permitted provided that the following

# conditions are met:

#

#   1. Redistributions of source code must retain the above

#      copyright notice, this list of conditions and the following

#      disclaimer.

#

#   2. Redistributions in binary form must reproduce the above

#      copyright notice, this list of conditions and the following

#      disclaimer in the documentation and/or other materials

#      provided with the distribution.

#

# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS

# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED

# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR

# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR

# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,

# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT

# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF

# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED

# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT

# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN

# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE

# POSSIBILITY OF SUCH DAMAGE.

#

# The views and conclusions contained in the software and

# documentation are those of the authors and should not be

# interpreted as representing official policies, either expressed

# or implied, of GRNET S.A.

#

set -e

rm -rf env

virtualenv --no-site-packages -ppython2.6 env

source env/bin/activate

export PIP_DOWNLOAD_CACHE=/tmp/.pip_cache

pip install -r requirements.pip

cd snf-common

rm -rf build dist

python setup.py install

cd ../snf-cyclades-app

rm -rf build dist

python setup.py install

cd ../snf-cyclades-gtools

rm -rf build dist

python setup.py install

cd ../env

# avoid vncauthproxy errors

rm bin/vncauthproxy.py

echo "running django tests..." >&2

export SYNNEFO_SETTINGS_DIR=/etc/lala

snf-manage test admin api db logic userdata --settings=synnefo.settings.test

cd ..

deactivate

#rm -rf env

#virtualenv --no-site-packages -ppython2.7 env

#source env/bin/activate

#export PIP_DOWNLOAD_CACHE=/tmp/.pip_cache

#pip install -r requirements.pip

#cd snf-common

#rm -rf build dist

#python setup.py install

#cd ../snf-cyclades-app

#rm -rf build dist

#python setup.py install

#cd ../snf-cyclades-gtools

#rm -rf build dist

#python setup.py install

#cd env

## avoid vncauthproxy errors

#rm bin/vncauthproxy.py

#echo "running django tests..." >&2

#snf-manage test aai admin api db helpdesk invitations logic userdata --settings=synnefo.settings.test

#cd ..

#deactivate

#rm -rf env

#!/bin/bash

#

#

# Copyright 2011 GRNET S.A. All rights reserved.

#

# Redistribution and use in source and binary forms, with or

# without modification, are permitted provided that the following

# conditions are met:

#

#   1. Redistributions of source code must retain the above

#      copyright notice, this list of conditions and the following

#      disclaimer.

#

#   2. Redistributions in binary form must reproduce the above

#      copyright notice, this list of conditions and the following

#      disclaimer in the documentation and/or other materials

#      provided with the distribution.

#

# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS

# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED

# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR

# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR

# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,

# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT

# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF

# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED

# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT

# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN

# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE

# POSSIBILITY OF SUCH DAMAGE.

#

# The views and conclusions contained in the software and

# documentation are those of the authors and should not be

# interpreted as representing official policies, either expressed

# or implied, of GRNET S.A.

#

set -e

echo "Running snf-cyclades-app tests..." >&2

snf-manage test admin api db logic userdata --settings=synnefo.settings.test

echo "Running snf-cyclades-gtools tests..." >&2

./snf-cyclades-gtools/test/synnefo.ganeti_unittest.py

#!/usr/bin/env python

#

# Copyright (C) 2010, 2011, 2012 GRNET S.A. All rights reserved.

#

# Redistribution and use in source and binary forms, with or

# without modification, are permitted provided that the following

# conditions are met:

#

#   1. Redistributions of source code must retain the above

#      copyright notice, this list of conditions and the following

#      disclaimer.

#

#   2. Redistributions in binary form must reproduce the above

#      copyright notice, this list of conditions and the following

#      disclaimer in the documentation and/or other materials

#      provided with the distribution.

#

# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS

# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED

# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR

# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR

# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,

# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT

# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF

# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED

# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT

# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN

# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE

# POSSIBILITY OF SUCH DAMAGE.

#

# The views and conclusions contained in the software and

# documentation are those of the authors and should not be

# interpreted as representing official policies, either expressed

# or implied, of GRNET S.A.

import os

import re

import sys

import pprint

import subprocess

import git

from distutils import log

from collections import namedtuple

# Branch types:

# builds_snapshot: Whether the branch can produce snapshot builds

# builds_release: Whether the branch can produce release builds

# versioned: Whether the name of the branch defines a specific version

# allowed_version_re: A regular expression describing allowed values for

#                     base_version in this branch

branch_type = namedtuple("branch_type", ["builds_snapshot", "builds_release",

                                         "versioned", "allowed_version_re"])

VERSION_RE = "[0-9]+\.[0-9]+"

BRANCH_TYPES = {

    "feature": branch_type(True, False, False, "^%snext$" % VERSION_RE),

    "develop": branch_type(True, False, False, "^%snext$" % VERSION_RE),

    "release": branch_type(True, True, True,

                           "^(?P<bverstr>%s)rc[1-9][0-9]*$" % VERSION_RE),

    "master": branch_type(False, True, False,

                          "^%s$" % VERSION_RE),

    "hotfix": branch_type(True, True, True,

                          "^(?P<bverstr>^%s\.[1-9][0-9]*)$" % VERSION_RE)}

BASE_VERSION_FILE = "version"

def get_commit_id(commit, current_branch):

    """Return the commit ID

    If the commit is a 'merge' commit, and one of the parents is a

    debian branch we return a compination of the parents commits.

    """

    def short_id(commit):

        return commit.hexsha[0:7]

    parents = commit.parents

    cur_br_name = current_branch.name

    if len(parents) == 1:

        return short_id(commit)

    elif len(parents) == 2:

        if cur_br_name.startswith("debian-") or cur_br_name == "debian":

            pr1, pr2 = parents

            return short_id(pr1) + "~" + short_id(pr2)

        else:

            return short_id(commit)

    else:

        raise RuntimeError("Commit %s has more than 2 parents!" % commit)

def vcs_info():

    """

    Return current git HEAD commit information.

    Returns a tuple containing

        - branch name

        - commit id

        - commit count

        - git describe output

        - path of git toplevel directory

    """

    try:

        repo = git.Repo(".")

        branch = repo.head.reference

        revid = get_commit_id(branch.commit, branch)

        revno = len(list(repo.iter_commits()))

        desc = repo.git.describe("--tags")

        toplevel = repo.working_dir

    except git.InvalidGitRepositoryError:

        log.error("Could not retrieve git information. " +

                  "Current directory not a git repository?")

        return None

    info = namedtuple("vcs_info", ["branch", "revid", "revno",

                                   "desc", "toplevel"])

    return info(branch=branch.name, revid=revid, revno=revno, desc=desc,

                toplevel=toplevel)

def base_version(vcs_info):

    """Determine the base version from a file in the repository"""

    f = open(os.path.join(vcs_info.toplevel, BASE_VERSION_FILE))

    lines = [l.strip() for l in f.readlines()]

    l = [l for l in lines if not l.startswith("#")]

    if len(l) != 1:

        raise ValueError("File '%s' should contain a single non-comment line.")

    return l[0]

def build_mode():

    """Determine the build mode from the value of $GITFLOW_BUILD_MODE"""

    try:

        mode = os.environ["GITFLOW_BUILD_MODE"]

        assert mode == "release" or mode == "snapshot"

    except (KeyError, AssertionError):

        raise ValueError("GITFLOW_BUILD_MODE environment variable must be "

                         "'release' or 'snapshot'")

    return mode

def python_version(base_version, vcs_info, mode):

    """Generate a Python distribution version following devtools conventions.

    This helper generates a Python distribution version from a repository

    commit, following devtools conventions. The input data are:

        * base_version: a base version number, presumably stored in text file

          inside the repository, e.g., /version

        * vcs_info: vcs information: current branch name and revision no

        * mode: "snapshot", or "release"

    This helper assumes a git branching model following:

    http://nvie.com/posts/a-successful-git-branching-model/

    with 'master', 'develop', 'release-X', 'hotfix-X' and 'feature-X' branches.

    General rules:

    a) any repository commit can get as a Python version

    b) a version is generated either in 'release' or in 'snapshot' mode

    c) the choice of mode depends on the branch, see following table.

    A python version is of the form A_NNN,

    where A: X.Y.Z{,next,rcW} and NNN: a revision number for the commit,

    as returned by vcs_info().

    For every combination of branch and mode, releases are numbered as follows:

    BRANCH:  /  MODE: snapshot        release

    --------          ------------------------------

    feature           0.14next_150    N/A

    develop           0.14next_151    N/A

    release           0.14rc2_249     0.14rc2

    master            N/A             0.14

    hotfix            0.14.1rc6_121   0.14.1rc6

                      N/A             0.14.1

    The suffix 'next' in a version name is used to denote the upcoming version,

    the one being under development in the develop and release branches.

    Version '0.14next' is the version following 0.14, and only lives on the

    develop and feature branches.

    The suffix 'rc' is used to denote release candidates. 'rc' versions live

    only in release and hotfix branches.

    Suffixes 'next' and 'rc' have been chosen to ensure proper ordering

    according to setuptools rules:

        http://www.python.org/dev/peps/pep-0386/#setuptools

    Every branch uses a value for A so that all releases are ordered based

    on the branch they came from, so:

    So

        0.13next < 0.14rcW < 0.14 < 0.14next < 0.14.1

    and

    >>> V("0.14next") > V("0.14")

    True

    >>> V("0.14next") > V("0.14rc7")

    True

    >>> V("0.14next") > V("0.14.1")

    False

    >>> V("0.14rc6") > V("0.14")

    False

    >>> V("0.14.2rc6") > V("0.14.1")

    True

    The value for _NNN is chosen based of the revision number of the specific

    commit. It is used to ensure ascending ordering of consecutive releases

    from the same branch. Every version of the form A_NNN comes *before*

    than A: All snapshots are ordered so they come before the corresponding

    release.

    So

        0.14next_* < 0.14

        0.14.1_* < 0.14.1

        etc

    and

    >>> V("0.14next_150") < V("0.14next")

    True

    >>> V("0.14.1next_150") < V("0.14.1next")

    True

    >>> V("0.14.1_149") < V("0.14.1")

    True

    >>> V("0.14.1_149") < V("0.14.1_150")

    True

    Combining both of the above, we get

       0.13next_* < 0.13next < 0.14rcW_* < 0.14rcW < 0.14_* < 0.14

       < 0.14next_* < 0.14next < 0.14.1_* < 0.14.1

    and

    >>> V("0.13next_102") < V("0.13next")

    True

    >>> V("0.13next") < V("0.14rc5_120")

    True

    >>> V("0.14rc3_120") < V("0.14rc3")

    True

    >>> V("0.14rc3") < V("0.14_1")

    True

    >>> V("0.14_120") < V("0.14")

    True

    >>> V("0.14") < V("0.14next_20")

    True

    >>> V("0.14next_20") < V("0.14next")

    True

    Note: one of the tests above fails because of constraints in the way

    setuptools parses version numbers. It does not affect us because the

    specific version format that triggers the problem is not contained in the

    table showing allowed branch / mode combinations, above.

    """

    branch = vcs_info.branch

    # If it's a debian branch, ignore starting "debian-"

    brnorm = branch

    if brnorm == "debian":

        brnorm = "debian-master"

    if brnorm.startswith("debian-"):

        brnorm = brnorm.replace("debian-", "", 1)

    # Sanity checks

    if "-" in brnorm:

        btypestr = brnorm.split("-")[0]

    else:

        btypestr = brnorm

    try:

        btype = BRANCH_TYPES[btypestr]

    except KeyError:

        allowed_branches = ", ".join(x for x in BRANCH_TYPES.keys())

        raise ValueError("Malformed branch name '%s', cannot classify as one "

                         "of %s" % (btypestr, allowed_branches))

    if btype.versioned:

        try:

            bverstr = brnorm.split("-")[1]

        except IndexError:

            # No version

            raise ValueError("Branch name '%s' should contain version" %

                             branch)

        # Check that version is well-formed

        if not re.match(VERSION_RE, bverstr):

            raise ValueError("Malformed version '%s' in branch name '%s'" %

                             (bverstr, branch))

    m = re.match(btype.allowed_version_re, base_version)

    if not m or (btype.versioned and m.groupdict()["bverstr"] != bverstr):

        raise ValueError("Base version '%s' unsuitable for branch name '%s'" %

                         (base_version, branch))

    if mode not in ["snapshot", "release"]:

        raise ValueError("Specified mode '%s' should be one of 'snapshot' or "

                         "'release'" % mode)

    snap = (mode == "snapshot")

    if ((snap and not btype.builds_snapshot) or

        (not snap and not btype.builds_release)):

        raise ValueError("Invalid mode '%s' in branch type '%s'" %

                         (mode, btypestr))

    if snap:

        v = "%s_%d_%s" % (base_version, vcs_info.revno, vcs_info.revid)

    else:

        v = base_version

    return v

def debian_version_from_python_version(pyver):

    """Generate a debian package version from a Python version.

    This helper generates a Debian package version from a Python version,

    following devtools conventions.

    Debian sorts version strings differently compared to setuptools:

    http://www.debian.org/doc/debian-policy/ch-controlfields.html#s-f-Version

    Initial tests:

    >>> debian_version("3") < debian_version("6")

    True

    >>> debian_version("3") < debian_version("2")

    False

    >>> debian_version("1") == debian_version("1")

    True

    >>> debian_version("1") != debian_version("1")

    False

    >>> debian_version("1") >= debian_version("1")

    True

    >>> debian_version("1") <= debian_version("1")

    True

    This helper defines a 1-1 mapping between Python and Debian versions,

    with the same ordering.

    Debian versions are ordered in the same way as Python versions:

    >>> D("0.14next") > D("0.14")

    True

    >>> D("0.14next") > D("0.14rc7")

    True

    >>> D("0.14next") > D("0.14.1")

    False

    >>> D("0.14rc6") > D("0.14")

    False

    >>> D("0.14.2rc6") > D("0.14.1")

    True

    and

    >>> D("0.14next_150") < D("0.14next")

    True

    >>> D("0.14.1next_150") < D("0.14.1next")

    True

    >>> D("0.14.1_149") < D("0.14.1")

    True

    >>> D("0.14.1_149") < D("0.14.1_150")

    True

    and

    >>> D("0.13next_102") < D("0.13next")

    True

    >>> D("0.13next") < D("0.14rc5_120")

    True

    >>> D("0.14rc3_120") < D("0.14rc3")

    True

    >>> D("0.14rc3") < D("0.14_1")

    True

    >>> D("0.14_120") < D("0.14")

    True

    >>> D("0.14") < D("0.14next_20")

    True

    >>> D("0.14next_20") < D("0.14next")

    True

    """

    return pyver.replace("_", "~").replace("rc", "~rc") + "-1"

def debian_version(base_version, vcs_info, mode):

    p = python_version(base_version, vcs_info, mode)

    return debian_version_from_python_version(p)

def user_info():

    import getpass

    import socket

    return "%s@%s" % (getpass.getuser(), socket.getfqdn())

def update_version(module, name="version", root="."):

    """

    Generate or replace version.py as a submodule of `module`.

    This is a helper to generate/replace a version.py file containing version

    information as a submodule of passed `module`.

    """

    v = vcs_info()

    if not v:

        # Return early if not in development environment

        return

    b = base_version(v)

    mode = build_mode()

    paths = [root] + module.split(".") + ["%s.py" % name]

    module_filename = os.path.join(*paths)

    version = python_version(b, v, mode)

    content = """

__version__ = "%(version)s"

__version_info__ = %(version_info)s

__version_vcs_info__ = %(vcs_info)s

__version_user_info__ = "%(user_info)s"

    """ % dict(version=version, version_info=version.split("."),

               vcs_info=pprint.PrettyPrinter().pformat(dict(v._asdict())),

               user_info=user_info())

    module_file = file(module_filename, "w+")

    module_file.write(content)

    module_file.close()

if __name__ == "__main__":