QA: adapt changes wrt DRBD usermode helper
[ganeti-local] / Makefile.am
index ffdee66..87d51c2 100644 (file)
@@ -20,6 +20,9 @@ strip_hsroot = $(patsubst src/%,%,$(patsubst test/hs/%,%,$(1)))
 # Use bash in order to be able to use pipefail
 SHELL=/bin/bash
 
+# Enable colors in shelltest
+SHELLTESTARGS = "-c"
+
 ACLOCAL_AMFLAGS = -I autotools
 BUILD_BASH_COMPLETION = $(top_srcdir)/autotools/build-bash-completion
 RUN_IN_TEMPDIR = $(top_srcdir)/autotools/run-in-tempdir
@@ -37,11 +40,23 @@ CONVERT_CONSTANTS = $(top_srcdir)/autotools/convert-constants
 BUILD_RPC = $(top_srcdir)/autotools/build-rpc
 SHELL_ENV_INIT = autotools/shell-env-init
 
+# starting as of Ganeti 2.10, all files are stored in a single directory,
+# with only symbolic links added at other places.
+versiondir = $(libdir)/ganeti-$(VERSION)
+
 # Note: these are automake-specific variables, and must be named after
 # the directory + 'dir' suffix
+pkglibdir = $(versiondir)$(libdir)/ganeti
+myexeclibdir = $(pkglibdir)
+bindir = $(versiondir)$(exec_prefix)/bin
+sbindir = $(versiondir)$(exec_prefix)/sbin
+mandir = $(versiondir)$(datarootdir)/man
+pkgpythondir = $(versiondir)$(pythondir)/ganeti
+
 clientdir = $(pkgpythondir)/client
 cmdlibdir = $(pkgpythondir)/cmdlib
 hypervisordir = $(pkgpythondir)/hypervisor
+storagedir = $(pkgpythondir)/storage
 httpdir = $(pkgpythondir)/http
 masterddir = $(pkgpythondir)/masterd
 confddir = $(pkgpythondir)/confd
@@ -53,8 +68,19 @@ utilsdir = $(pkgpythondir)/utils
 toolsdir = $(pkglibdir)/tools
 iallocatorsdir = $(pkglibdir)/iallocators
 pytoolsdir = $(pkgpythondir)/tools
-docdir = $(datadir)/doc/$(PACKAGE)
-myexeclibdir = $(pkglibdir)
+docdir = $(versiondir)$(datadir)/doc/$(PACKAGE)
+
+SYMLINK_TARGET_DIRS = \
+       $(pythondir) \
+       $(libdir)/ganeti/iallocators \
+       $(exec_prefix)/bin \
+       $(exec_prefix)/sbin \
+       $(datarootdir)/man/man1 \
+       $(datarootdir)/man/man7 \
+       $(datarootdir)/man/man8
+SYMLINK_DIRS = \
+       $(pythondir)/ganeti \
+       $(libdir)/ganeti/tools
 
 # Delete output file if an error occurred while building it
 .DELETE_ON_ERROR:
@@ -62,11 +88,11 @@ myexeclibdir = $(pkglibdir)
 HS_DIRS = \
        src \
        src/Ganeti \
-       src/Ganeti/Block \
-       src/Ganeti/Block/Drbd \
        src/Ganeti/Confd \
        src/Ganeti/Curl \
+       src/Ganeti/Cpu \
        src/Ganeti/DataCollectors \
+       src/Ganeti/Hs2Py \
        src/Ganeti/HTools \
        src/Ganeti/HTools/Backend \
        src/Ganeti/HTools/Program \
@@ -74,11 +100,17 @@ HS_DIRS = \
        src/Ganeti/Hypervisor/Xen \
        src/Ganeti/Monitoring \
        src/Ganeti/Query \
+       src/Ganeti/Storage \
+       src/Ganeti/Storage/Diskstats \
+       src/Ganeti/Storage/Drbd \
+       src/Ganeti/Storage/Lvm \
        test/hs \
        test/hs/Test \
        test/hs/Test/Ganeti \
-       test/hs/Test/Ganeti/Block \
-       test/hs/Test/Ganeti/Block/Drbd \
+       test/hs/Test/Ganeti/Storage \
+       test/hs/Test/Ganeti/Storage/Diskstats \
+       test/hs/Test/Ganeti/Storage/Drbd \
+       test/hs/Test/Ganeti/Storage/Lvm \
        test/hs/Test/Ganeti/Confd \
        test/hs/Test/Ganeti/HTools \
        test/hs/Test/Ganeti/HTools/Backend \
@@ -103,6 +135,7 @@ DIRS = \
        test/data/htools \
        test/data/htools/rapi \
        test/hs/shelltests \
+       test/autotools \
        lib \
        lib/build \
        lib/client \
@@ -114,6 +147,7 @@ DIRS = \
        lib/masterd \
        lib/rapi \
        lib/server \
+       lib/storage \
        lib/tools \
        lib/utils \
        lib/watcher \
@@ -125,6 +159,8 @@ DIRS = \
        test/data/ovfdata \
        test/data/ovfdata/other \
        test/py \
+       test/py/cmdlib \
+       test/py/cmdlib/testsupport \
        tools
 
 ALL_APIDOC_HS_DIRS = \
@@ -192,6 +228,8 @@ CLEANFILES = \
        $(man_MANS) \
        $(manhtml) \
        tools/kvm-ifup \
+       tools/vif-ganeti \
+       tools/net-common \
        tools/users-setup \
        tools/vcluster-setup \
        stamp-directories \
@@ -200,6 +238,7 @@ CLEANFILES = \
        $(HS_ALL_PROGS) $(HS_BUILT_SRCS) \
        $(HS_BUILT_TEST_HELPERS) \
        src/ganeti-confd \
+       src/ganeti-luxid \
        src/ganeti-mond \
        .hpc/*.mix src/*.tix test/hs/*.tix \
        doc/hs-lint.html
@@ -213,7 +252,7 @@ HS_GENERATED_FILES =
 if WANT_HTOOLS
 HS_GENERATED_FILES += $(HS_PROGS)
 if ENABLE_CONFD
-HS_GENERATED_FILES += src/hconfd src/ganeti-confd
+HS_GENERATED_FILES += src/hconfd src/ganeti-confd src/hluxid src/ganeti-luxid
 endif
 
 if ENABLE_MOND
@@ -227,7 +266,8 @@ built_base_sources = \
 
 built_python_base_sources = \
        lib/_autoconf.py \
-       lib/_vcsversion.py
+       lib/_vcsversion.py \
+       lib/opcodes.py
 
 BUILT_PYTHON_SOURCES = \
        $(built_python_base_sources) \
@@ -242,6 +282,7 @@ BUILT_EXAMPLES = \
        doc/examples/ganeti-kvm-poweroff.initd \
        doc/examples/ganeti.cron \
        doc/examples/ganeti.initd \
+       doc/examples/ganeti.logrotate \
        doc/examples/ganeti-master-role.ocf \
        doc/examples/ganeti-node-role.ocf \
        doc/examples/gnt-config-backup \
@@ -259,7 +300,6 @@ pkgpython_PYTHON = \
        lib/__init__.py \
        lib/asyncnotifier.py \
        lib/backend.py \
-       lib/bdev.py \
        lib/bootstrap.py \
        lib/cli.py \
        lib/compat.py \
@@ -276,7 +316,7 @@ pkgpython_PYTHON = \
        lib/mcpu.py \
        lib/netutils.py \
        lib/objects.py \
-       lib/opcodes.py \
+       lib/opcodes_base.py \
        lib/outils.py \
        lib/ovf.py \
        lib/pathutils.py \
@@ -288,7 +328,6 @@ pkgpython_PYTHON = \
        lib/serializer.py \
        lib/ssconf.py \
        lib/ssh.py \
-       lib/storage.py \
        lib/uidpool.py \
        lib/vcluster.py \
        lib/network.py \
@@ -337,6 +376,16 @@ hypervisor_PYTHON = \
        lib/hypervisor/hv_lxc.py \
        lib/hypervisor/hv_xen.py
 
+storage_PYTHON = \
+       lib/storage/__init__.py \
+       lib/storage/bdev.py \
+       lib/storage/base.py \
+       lib/storage/container.py \
+       lib/storage/drbd.py \
+       lib/storage/drbd_info.py \
+       lib/storage/drbd_cmdgen.py \
+       lib/storage/filestorage.py
+
 rapi_PYTHON = \
        lib/rapi/__init__.py \
        lib/rapi/baserlib.py \
@@ -395,6 +444,7 @@ utils_PYTHON = \
        lib/utils/nodesetup.py \
        lib/utils/process.py \
        lib/utils/retry.py \
+       lib/utils/storage.py \
        lib/utils/text.py \
        lib/utils/wrapper.py \
        lib/utils/x509.py
@@ -412,15 +462,18 @@ docinput = \
        doc/design-2.5.rst \
        doc/design-2.6.rst \
        doc/design-2.7.rst \
+       doc/design-2.8.rst \
        doc/design-autorepair.rst \
        doc/design-bulk-create.rst \
        doc/design-chained-jobs.rst \
        doc/design-cpu-pinning.rst \
        doc/design-device-uuid-name.rst \
        doc/design-draft.rst \
+       doc/design-hotplug.rst \
        doc/design-htools-2.3.rst \
        doc/design-http-server.rst \
        doc/design-impexp2.rst \
+       doc/design-internal-shutdown.rst \
        doc/design-lu-generated-jobs.rst \
        doc/design-linuxha.rst \
        doc/design-multi-reloc.rst \
@@ -429,6 +482,7 @@ docinput = \
        doc/design-oob.rst \
        doc/design-ovf-support.rst \
        doc/design-opportunistic-locking.rst \
+       doc/design-optables.rst \
        doc/design-partitioned.rst \
        doc/design-query-splitting.rst \
        doc/design-query2.rst \
@@ -441,6 +495,7 @@ docinput = \
        doc/design-x509-ca.rst \
        doc/design-hroller.rst \
        doc/design-storagetypes.rst \
+        doc/design-upgrade.rst \
        doc/devnotes.rst \
        doc/glossary.rst \
        doc/hooks.rst \
@@ -450,6 +505,7 @@ docinput = \
        doc/install.rst \
        doc/locking.rst \
        doc/manpages-disabled.rst \
+       doc/monitoring-query-format.rst \
        doc/move-instance.rst \
        doc/news.rst \
        doc/ovfconverter.rst \
@@ -476,6 +532,8 @@ endif
 HS_COMPILE_PROGS= \
        src/ganeti-mond \
        src/hconfd \
+       src/hluxid \
+       src/hs2py \
        src/rpc-test
 
 # All Haskell non-test programs to be compiled but not automatically installed
@@ -518,20 +576,27 @@ HPCEXCL = --exclude Main \
        $(patsubst src.%,--exclude Test.%,$(subst /,.,$(patsubst %.hs,%, $(HS_LIB_SRCS))))
 
 HS_LIB_SRCS = \
-       src/Ganeti/Block/Drbd/Types.hs \
-       src/Ganeti/Block/Drbd/Parser.hs \
        src/Ganeti/BasicTypes.hs \
        src/Ganeti/Common.hs \
        src/Ganeti/Compat.hs \
        src/Ganeti/Confd/Client.hs \
+       src/Ganeti/Confd/ClientFunctions.hs \
        src/Ganeti/Confd/Server.hs \
        src/Ganeti/Confd/Types.hs \
        src/Ganeti/Confd/Utils.hs \
        src/Ganeti/Config.hs \
+       src/Ganeti/ConfigReader.hs \
+       src/Ganeti/Cpu/LoadParser.hs \
+       src/Ganeti/Cpu/Types.hs \
        src/Ganeti/Curl/Multi.hs \
        src/Ganeti/Daemon.hs \
        src/Ganeti/DataCollectors/CLI.hs \
+       src/Ganeti/DataCollectors/CPUload.hs \
+       src/Ganeti/DataCollectors/Diskstats.hs \
        src/Ganeti/DataCollectors/Drbd.hs \
+       src/Ganeti/DataCollectors/InstStatus.hs \
+       src/Ganeti/DataCollectors/InstStatusTypes.hs \
+       src/Ganeti/DataCollectors/Lv.hs \
        src/Ganeti/DataCollectors/Program.hs \
        src/Ganeti/DataCollectors/Types.hs \
        src/Ganeti/Errors.hs \
@@ -548,6 +613,7 @@ HS_LIB_SRCS = \
        src/Ganeti/HTools/Group.hs \
        src/Ganeti/HTools/Instance.hs \
        src/Ganeti/HTools/Loader.hs \
+       src/Ganeti/HTools/Nic.hs \
        src/Ganeti/HTools/Node.hs \
        src/Ganeti/HTools/PeerMap.hs \
        src/Ganeti/HTools/Program/Hail.hs \
@@ -560,9 +626,12 @@ HS_LIB_SRCS = \
        src/Ganeti/HTools/Program/Hroller.hs \
        src/Ganeti/HTools/Program/Main.hs \
        src/Ganeti/HTools/Types.hs \
+       src/Ganeti/Hypervisor/Xen.hs \
        src/Ganeti/Hypervisor/Xen/XmParser.hs \
        src/Ganeti/Hypervisor/Xen/Types.hs \
        src/Ganeti/Hash.hs \
+       src/Ganeti/Hs2Py/GenOpCodes.hs \
+       src/Ganeti/Hs2Py/OpDoc.hs \
        src/Ganeti/JQueue.hs \
        src/Ganeti/JSON.hs \
        src/Ganeti/Jobs.hs \
@@ -574,6 +643,8 @@ HS_LIB_SRCS = \
        src/Ganeti/OpCodes.hs \
        src/Ganeti/OpParams.hs \
        src/Ganeti/Path.hs \
+       src/Ganeti/Parsers.hs \
+       src/Ganeti/Query/Cluster.hs \
        src/Ganeti/Query/Common.hs \
        src/Ganeti/Query/Export.hs \
        src/Ganeti/Query/Filter.hs \
@@ -588,6 +659,13 @@ HS_LIB_SRCS = \
        src/Ganeti/Rpc.hs \
        src/Ganeti/Runtime.hs \
        src/Ganeti/Ssconf.hs \
+       src/Ganeti/Storage/Diskstats/Parser.hs \
+       src/Ganeti/Storage/Diskstats/Types.hs \
+       src/Ganeti/Storage/Drbd/Parser.hs \
+       src/Ganeti/Storage/Drbd/Types.hs \
+       src/Ganeti/Storage/Lvm/LVParser.hs \
+       src/Ganeti/Storage/Lvm/Types.hs \
+       src/Ganeti/Storage/Utils.hs \
        src/Ganeti/THH.hs \
        src/Ganeti/Types.hs \
        src/Ganeti/Utils.hs
@@ -595,8 +673,6 @@ HS_LIB_SRCS = \
 HS_TEST_SRCS = \
        test/hs/Test/Ganeti/Attoparsec.hs \
        test/hs/Test/Ganeti/BasicTypes.hs \
-       test/hs/Test/Ganeti/Block/Drbd/Parser.hs \
-       test/hs/Test/Ganeti/Block/Drbd/Types.hs \
        test/hs/Test/Ganeti/Common.hs \
        test/hs/Test/Ganeti/Confd/Types.hs \
        test/hs/Test/Ganeti/Confd/Utils.hs \
@@ -628,6 +704,10 @@ HS_TEST_SRCS = \
        test/hs/Test/Ganeti/Rpc.hs \
        test/hs/Test/Ganeti/Runtime.hs \
        test/hs/Test/Ganeti/Ssconf.hs \
+       test/hs/Test/Ganeti/Storage/Diskstats/Parser.hs \
+       test/hs/Test/Ganeti/Storage/Drbd/Parser.hs \
+       test/hs/Test/Ganeti/Storage/Drbd/Types.hs \
+       test/hs/Test/Ganeti/Storage/Lvm/LVParser.hs \
        test/hs/Test/Ganeti/THH.hs \
        test/hs/Test/Ganeti/TestCommon.hs \
        test/hs/Test/Ganeti/TestHTools.hs \
@@ -644,7 +724,9 @@ HS_BUILT_SRCS = \
        src/Ganeti/Version.hs
 HS_BUILT_SRCS_IN = \
        $(patsubst %,%.in,$(filter-out src/Ganeti/Curl/Internal.hs,$(HS_BUILT_SRCS))) \
-       src/Ganeti/Curl/Internal.hsc
+       src/Ganeti/Curl/Internal.hsc \
+       lib/opcodes.py.in_after \
+       lib/opcodes.py.in_before
 
 HS_LIBTESTBUILT_SRCS = $(HS_LIBTEST_SRCS) $(HS_BUILT_SRCS)
 
@@ -661,9 +743,9 @@ doc/man-html/index.html: doc/manpages-enabled.rst $(mandocrst)
 # it changes
 doc/html/index.html doc/man-html/index.html: $(docinput) doc/conf.py \
        configure.ac $(RUN_IN_TEMPDIR) lib/build/sphinx_ext.py \
-       lib/build/shell_example_lexer.py lib/opcodes.py lib/ht.py \
+       lib/build/shell_example_lexer.py lib/ht.py \
        doc/css/style.css lib/rapi/connector.py lib/rapi/rlib2.py \
-       | $(BUILT_PYTHON_SOURCES)
+       autotools/sphinx-wrapper | $(BUILT_PYTHON_SOURCES)
        @test -n "$(SPHINX)" || \
            { echo 'sphinx-build' not found during configure; exit 1; }
 if !MANPAGES_IN_DOC
@@ -738,14 +820,17 @@ endif
 # Things to build but not to install (add it to EXTRA_DIST if it should be
 # distributed)
 noinst_DATA = \
-       doc/html \
        $(BUILT_EXAMPLES) \
        doc/examples/bash_completion \
        doc/examples/bash_completion-debug \
        $(manhtml)
 
+if HAS_SPHINX
 if MANPAGES_IN_DOC
 noinst_DATA += doc/man-html
+else
+noinst_DATA += doc/html
+endif
 endif
 
 gnt_scripts = \
@@ -785,7 +870,9 @@ qa_scripts = \
        qa/qa_error.py \
        qa/qa_group.py \
        qa/qa_instance.py \
+       qa/qa_instance_utils.py \
        qa/qa_job.py \
+       qa/qa_monitoring.py \
        qa/qa_node.py \
        qa/qa_os.py \
        qa/qa_rapi.py \
@@ -829,7 +916,7 @@ test/hs/hpc-htools: HEXTRA_INT=-fhpc
 test/hs/hpc-mon-collector: HEXTRA_INT=-fhpc
 
 # test dependency
-test/hs/offline-tests.sh: test/hs/hpc-htools test/hs/hpc-mon-collector
+test/hs/offline-test.sh: test/hs/hpc-htools test/hs/hpc-mon-collector
 
 # rules for building profiling-enabled versions of the haskell
 # programs: hs-prof does the full two-step build, whereas
@@ -860,11 +947,20 @@ nodist_sbin_SCRIPTS = \
        $(PYTHON_BOOTSTRAP_SBIN) \
        daemons/ganeti-cleaner
 
+# strip path prefixes off the sbin scripts
+all_sbin_scripts = \
+       $(patsubst tools/%,%,$(patsubst daemons/%,%,$(patsubst scripts/%,%,\
+       $(patsubst src/%,%,$(dist_sbin_SCRIPTS) $(nodist_sbin_SCRIPTS)))))
+
 if ENABLE_CONFD
 src/ganeti-confd: src/hconfd
        cp -f $< $@
 
+src/ganeti-luxid: src/hluxid
+       cp -f $< $@
+
 nodist_sbin_SCRIPTS += src/ganeti-confd
+nodist_sbin_SCRIPTS += src/ganeti-luxid
 endif
 
 if ENABLE_MOND
@@ -910,9 +1006,16 @@ nodist_pkglib_python_scripts = \
 myexeclib_SCRIPTS = \
        daemons/daemon-util \
        tools/kvm-ifup \
+       tools/vif-ganeti \
+       tools/net-common \
        $(pkglib_python_scripts) \
        $(HS_MYEXECLIB_PROGS)
 
+# strip path prefixes off the sbin scripts
+all_myexeclib_scripts = \
+       $(patsubst tools/%,%,$(patsubst daemons/%,%,\
+       $(patsubst src/%,%,$(myexeclib_SCRIPTS))))
+
 nodist_myexeclib_SCRIPTS = \
        $(nodist_pkglib_python_scripts)
 
@@ -921,6 +1024,7 @@ EXTRA_DIST = \
        UPGRADE \
        epydoc.conf.in \
        pylintrc \
+       pylintrc-test \
        autotools/build-bash-completion \
        autotools/build-rpc \
        autotools/check-header \
@@ -945,6 +1049,8 @@ EXTRA_DIST = \
        devel/upload \
        devel/webserver \
        tools/kvm-ifup.in \
+       tools/vif-ganeti.in \
+       tools/net-common.in \
        tools/users-setup.in \
        tools/vcluster-setup.in \
        $(docinput) \
@@ -956,11 +1062,9 @@ EXTRA_DIST = \
        doc/examples/gnt-debug/README \
        doc/examples/gnt-debug/delay0.json \
        doc/examples/gnt-debug/delay50.json \
-       test/py/lockperf.py \
-       test/py/testutils.py \
-       test/py/mocks.py \
        $(dist_TESTS) \
        $(TEST_FILES) \
+       $(python_test_support) \
        man/footer.rst \
        $(manrst) \
        $(maninput) \
@@ -976,6 +1080,7 @@ EXTRA_DIST = \
 man_MANS = \
        man/ganeti-cleaner.8 \
        man/ganeti-confd.8 \
+       man/ganeti-luxid.8 \
        man/ganeti-listrunner.8 \
        man/ganeti-masterd.8 \
        man/ganeti-mond.8 \
@@ -1017,12 +1122,20 @@ maninput = \
        $(patsubst %.html,%.html.in,$(manhtml)) \
        $(mangen)
 
+manfullpath = $(patsubst man/%.1,man1/%.1,\
+       $(patsubst man/%.7,man7/%.7,\
+       $(patsubst man/%.8,man8/%.8,$(man_MANS))))
+
 TEST_FILES = \
+       test/autotools/autotools-check-news.test \
        test/data/htools/clean-nonzero-score.data \
        test/data/htools/common-suffix.data \
        test/data/htools/empty-cluster.data \
        test/data/htools/hail-alloc-drbd.json \
+       test/data/htools/hail-alloc-invalid-network.json \
        test/data/htools/hail-alloc-invalid-twodisks.json \
+       test/data/htools/hail-alloc-restricted-network.json \
+       test/data/htools/hail-alloc-spindles.json \
        test/data/htools/hail-alloc-twodisks.json \
        test/data/htools/hail-change-group.json \
        test/data/htools/hail-invalid-reloc.json \
@@ -1030,16 +1143,27 @@ TEST_FILES = \
        test/data/htools/hail-reloc-drbd.json \
        test/data/htools/hbal-excl-tags.data \
        test/data/htools/hbal-split-insts.data \
+       test/data/htools/hspace-tiered-dualspec-exclusive.data \
        test/data/htools/hspace-tiered-dualspec.data \
+       test/data/htools/hspace-tiered-exclusive.data \
+       test/data/htools/hspace-tiered-ipolicy.data \
+       test/data/htools/hspace-tiered-mixed.data \
+       test/data/htools/hspace-tiered-resourcetypes.data \
+       test/data/htools/hspace-tiered-vcpu.data \
        test/data/htools/hspace-tiered.data \
        test/data/htools/invalid-node.data \
        test/data/htools/missing-resources.data \
        test/data/htools/multiple-master.data \
+       test/data/htools/multiple-tags.data \
        test/data/htools/n1-failure.data \
        test/data/htools/rapi/groups.json \
        test/data/htools/rapi/info.json \
        test/data/htools/rapi/instances.json \
        test/data/htools/rapi/nodes.json \
+       test/data/htools/hroller-full.data \
+       test/data/htools/hroller-nodegroups.data \
+       test/data/htools/hroller-nonredundant.data \
+       test/data/htools/hroller-online.data \
        test/data/htools/unique-reboot-order.data \
        test/hs/shelltests/htools-balancing.test \
        test/hs/shelltests/htools-basic.test \
@@ -1057,6 +1181,8 @@ TEST_FILES = \
        test/hs/shelltests/htools-mon-collector.test \
        test/data/bdev-drbd-8.0.txt \
        test/data/bdev-drbd-8.3.txt \
+       test/data/bdev-drbd-8.4.txt \
+       test/data/bdev-drbd-8.4-no-disk-params.txt \
        test/data/bdev-drbd-disk.txt \
        test/data/bdev-drbd-net-ip4.txt \
        test/data/bdev-drbd-net-ip6.txt \
@@ -1075,8 +1201,10 @@ TEST_FILES = \
        test/data/cert1.pem \
        test/data/cert2.pem \
        test/data/cluster_config_2.7.json \
-       test/data/cluster_config_downgraded_2.7.json \
+       test/data/cluster_config_2.8.json \
+       test/data/cluster_config_2.9.json \
        test/data/instance-minor-pairing.txt \
+       test/data/instance-prim-sec.txt \
        test/data/ip-addr-show-dummy0.txt \
        test/data/ip-addr-show-lo-ipv4.txt \
        test/data/ip-addr-show-lo-ipv6.txt \
@@ -1090,6 +1218,9 @@ TEST_FILES = \
        test/data/kvm_0.9.1_help_boot_test.txt \
        test/data/kvm_1.0_help.txt \
        test/data/kvm_1.1.2_help.txt \
+       test/data/lvs_lv.txt \
+       test/data/NEWS_OK.txt \
+       test/data/NEWS_previous_unreleased.txt \
        test/data/ovfdata/compr_disk.vmdk.gz \
        test/data/ovfdata/config.ini \
        test/data/ovfdata/corrupted_resources.ovf \
@@ -1115,12 +1246,16 @@ TEST_FILES = \
        test/data/ovfdata/wrong_manifest.ovf \
        test/data/ovfdata/wrong_ova.ova \
        test/data/ovfdata/wrong_xml.ovf \
+       test/data/proc_diskstats.txt \
        test/data/proc_drbd8.txt \
        test/data/proc_drbd80-emptyline.txt \
+       test/data/proc_drbd80-emptyversion.txt \
        test/data/proc_drbd83.txt \
        test/data/proc_drbd83_sync.txt \
        test/data/proc_drbd83_sync_want.txt \
        test/data/proc_drbd83_sync_krnl2.6.39.txt \
+       test/data/proc_drbd84.txt \
+       test/data/proc_drbd84_sync.txt \
        test/data/qa-minimal-nodes-instances-only.json \
        test/data/sys_drbd_usermode_helper.txt \
        test/data/vgreduce-removemissing-2.02.02.txt \
@@ -1137,19 +1272,25 @@ TEST_FILES = \
        test/py/gnt-cli.test \
        test/py/import-export_unittest-helper
 
+
 python_tests = \
        doc/examples/rapi_testutils.py \
+       test/py/cmdlib/cluster_unittest.py \
+       test/py/cmdlib/cmdlib_unittest.py \
+       test/py/cmdlib/group_unittest.py \
+       test/py/cmdlib/instance_unittest.py \
+       test/py/cmdlib/instance_storage_unittest.py \
+       test/py/cmdlib/test_unittest.py \
        test/py/cfgupgrade_unittest.py \
        test/py/docs_unittest.py \
        test/py/ganeti.asyncnotifier_unittest.py \
        test/py/ganeti.backend_unittest-runasroot.py \
        test/py/ganeti.backend_unittest.py \
-       test/py/ganeti.bdev_unittest.py \
+       test/py/ganeti.bootstrap_unittest.py \
        test/py/ganeti.cli_unittest.py \
        test/py/ganeti.client.gnt_cluster_unittest.py \
        test/py/ganeti.client.gnt_instance_unittest.py \
        test/py/ganeti.client.gnt_job_unittest.py \
-       test/py/ganeti.cmdlib_unittest.py \
        test/py/ganeti.compat_unittest.py \
        test/py/ganeti.confd.client_unittest.py \
        test/py/ganeti.config_unittest.py \
@@ -1191,7 +1332,10 @@ python_tests = \
        test/py/ganeti.server.rapi_unittest.py \
        test/py/ganeti.ssconf_unittest.py \
        test/py/ganeti.ssh_unittest.py \
-       test/py/ganeti.storage_unittest.py \
+       test/py/ganeti.storage.bdev_unittest.py \
+       test/py/ganeti.storage.container_unittest.py \
+       test/py/ganeti.storage.drbd_unittest.py \
+       test/py/ganeti.storage.filestorage_unittest.py \
        test/py/ganeti.tools.burnin_unittest.py \
        test/py/ganeti.tools.ensure_dirs_unittest.py \
        test/py/ganeti.tools.node_daemon_setup_unittest.py \
@@ -1208,6 +1352,7 @@ python_tests = \
        test/py/ganeti.utils.nodesetup_unittest.py \
        test/py/ganeti.utils.process_unittest.py \
        test/py/ganeti.utils.retry_unittest.py \
+       test/py/ganeti.utils.storage_unittest.py \
        test/py/ganeti.utils.text_unittest.py \
        test/py/ganeti.utils.wrapper_unittest.py \
        test/py/ganeti.utils.x509_unittest.py \
@@ -1218,6 +1363,23 @@ python_tests = \
        test/py/qa.qa_config_unittest.py \
        test/py/tempfile_fork_unittest.py
 
+python_test_support = \
+       test/py/__init__.py \
+       test/py/lockperf.py \
+       test/py/testutils.py \
+       test/py/mocks.py \
+       test/py/cmdlib/__init__.py \
+       test/py/cmdlib/testsupport/__init__.py \
+       test/py/cmdlib/testsupport/cmdlib_testcase.py \
+       test/py/cmdlib/testsupport/config_mock.py \
+       test/py/cmdlib/testsupport/iallocator_mock.py \
+       test/py/cmdlib/testsupport/lock_manager_mock.py \
+       test/py/cmdlib/testsupport/netutils_mock.py \
+       test/py/cmdlib/testsupport/processor_mock.py \
+       test/py/cmdlib/testsupport/rpc_runner_mock.py \
+       test/py/cmdlib/testsupport/ssh_mock.py \
+       test/py/cmdlib/testsupport/util.py
+
 haskell_tests = test/hs/htest
 
 dist_TESTS = \
@@ -1226,8 +1388,11 @@ dist_TESTS = \
        test/py/ganeti-cleaner_unittest.bash \
        test/py/import-export_unittest.bash \
        test/py/cli-test.bash \
-       test/py/bash_completion.bash \
-       $(python_tests)
+       test/py/bash_completion.bash
+
+if PY_UNIT
+dist_TESTS += $(python_tests)
+endif
 
 nodist_TESTS =
 check_SCRIPTS =
@@ -1245,7 +1410,7 @@ TESTS = $(dist_TESTS) $(nodist_TESTS)
 
 # Environment for all tests
 PLAIN_TESTS_ENVIRONMENT = \
-       PYTHONPATH=. \
+       PYTHONPATH=.:./test/py \
        TOP_SRCDIR=$(abs_top_srcdir) TOP_BUILDDIR=$(abs_top_builddir) \
        PYTHON=$(PYTHON) FAKEROOT=$(FAKEROOT_PATH) \
        $(RUN_IN_TEMPDIR)
@@ -1260,11 +1425,11 @@ all_python_code = \
        $(pkglib_python_scripts) \
        $(nodist_pkglib_python_scripts) \
        $(nodist_tools_python_scripts) \
-       $(python_tests) \
        $(pkgpython_PYTHON) \
        $(client_PYTHON) \
        $(cmdlib_PYTHON) \
        $(hypervisor_PYTHON) \
+       $(storage_PYTHON) \
        $(rapi_PYTHON) \
        $(server_PYTHON) \
        $(pytools_PYTHON) \
@@ -1277,6 +1442,11 @@ all_python_code = \
        $(noinst_PYTHON) \
        $(qa_scripts)
 
+if PY_UNIT
+all_python_code += $(python_tests)
+all_python_code += $(python_test_support)
+endif
+
 srclink_files = \
        man/footer.rst \
        test/py/check-cert-expired_unittest.bash \
@@ -1324,7 +1494,8 @@ pep8_python_code = \
        $(CHECK_HEADER) \
        $(DOCPP) \
        $(PYTHON_BOOTSTRAP) \
-       qa
+       qa \
+       $(python_test_support)
 
 test/py/daemon-util_unittest.bash: daemons/daemon-util
 
@@ -1336,6 +1507,14 @@ tools/kvm-ifup: tools/kvm-ifup.in $(REPLACE_VARS_SED)
        sed -f $(REPLACE_VARS_SED) < $< > $@
        chmod +x $@
 
+tools/vif-ganeti: tools/vif-ganeti.in $(REPLACE_VARS_SED)
+       sed -f $(REPLACE_VARS_SED) < $< > $@
+       chmod +x $@
+
+tools/net-common: tools/net-common.in $(REPLACE_VARS_SED)
+       sed -f $(REPLACE_VARS_SED) < $< > $@
+       chmod +x $@
+
 tools/users-setup: tools/users-setup.in $(REPLACE_VARS_SED)
        sed -f $(REPLACE_VARS_SED) < $< > $@
        chmod +x $@
@@ -1488,10 +1667,7 @@ lib/_autoconf.py: Makefile | stamp-directories
          echo "XEN_KERNEL = '$(XEN_KERNEL)'"; \
          echo "XEN_INITRD = '$(XEN_INITRD)'"; \
          echo "KVM_KERNEL = '$(KVM_KERNEL)'"; \
-         echo "FILE_STORAGE_DIR = '$(FILE_STORAGE_DIR)'"; \
-         echo "ENABLE_FILE_STORAGE = $(ENABLE_FILE_STORAGE)"; \
          echo "SHARED_FILE_STORAGE_DIR = '$(SHARED_FILE_STORAGE_DIR)'"; \
-         echo "ENABLE_SHARED_FILE_STORAGE = $(ENABLE_SHARED_FILE_STORAGE)"; \
          echo "IALLOCATOR_SEARCH_PATH = [$(IALLOCATOR_SEARCH_PATH)]"; \
          echo "KVM_PATH = '$(KVM_PATH)'"; \
          echo "IP_PATH = '$(IP_PATH)'"; \
@@ -1514,6 +1690,8 @@ lib/_autoconf.py: Makefile | stamp-directories
          echo "RAPI_GROUP = '$(RAPI_GROUP)'"; \
          echo "CONFD_USER = '$(CONFD_USER)'"; \
          echo "CONFD_GROUP = '$(CONFD_GROUP)'"; \
+         echo "LUXID_USER = '$(LUXID_USER)'"; \
+         echo "LUXID_GROUP = '$(LUXID_GROUP)'"; \
          echo "NODED_USER = '$(NODED_USER)'"; \
          echo "NODED_GROUP = '$(NODED_GROUP)'"; \
          echo "MOND_USER = '$(MOND_USER)'"; \
@@ -1556,6 +1734,13 @@ lib/_vcsversion.py: Makefile vcs-version | stamp-directories
          echo "VCS_VERSION = '$$VCSVER'"; \
        } > $@
 
+lib/opcodes.py: Makefile src/hs2py src/Ganeti/Constants.hs \
+               lib/opcodes.py.in_before lib/opcodes.py.in_after \
+               | stamp-directories
+       cat $(abs_top_srcdir)/lib/opcodes.py.in_before > $@
+       src/hs2py >> $@
+       cat $(abs_top_srcdir)/lib/opcodes.py.in_after >> $@
+
 lib/_generated_rpc.py: lib/rpc_defs.py $(BUILD_RPC)
        PYTHONPATH=. $(RUN_IN_TEMPDIR) $(CURDIR)/$(BUILD_RPC) lib/rpc_defs.py > $@
 
@@ -1580,8 +1765,8 @@ $(REPLACE_VARS_SED): $(SHELL_ENV_INIT) Makefile stamp-directories
        { echo 's#@''PREFIX@#$(prefix)#g'; \
          echo 's#@''SYSCONFDIR@#$(sysconfdir)#g'; \
          echo 's#@''LOCALSTATEDIR@#$(localstatedir)#g'; \
-         echo 's#@''BINDIR@#$(bindir)#g'; \
-         echo 's#@''SBINDIR@#$(sbindir)#g'; \
+         echo 's#@''BINDIR@#$(execprefix)/bin#g'; \
+         echo 's#@''SBINDIR@#$(execprefix)/sbin#g'; \
          echo 's#@''LIBDIR@#$(libdir)#g'; \
          echo 's#@''GANETI_VERSION@#$(PACKAGE_VERSION)#g'; \
          echo 's#@''CUSTOM_XEN_BOOTLOADER@#$(XEN_BOOTLOADER)#g'; \
@@ -1589,17 +1774,18 @@ $(REPLACE_VARS_SED): $(SHELL_ENV_INIT) Makefile stamp-directories
          echo 's#@''CUSTOM_XEN_INITRD@#$(XEN_INITRD)#g'; \
          echo 's#@''CUSTOM_IALLOCATOR_SEARCH_PATH@#$(IALLOCATOR_SEARCH_PATH)#g'; \
          echo 's#@''CUSTOM_EXPORT_DIR@#$(EXPORT_DIR)#g'; \
-         echo 's#@''RPL_FILE_STORAGE_DIR@#$(FILE_STORAGE_DIR)#g'; \
          echo 's#@''RPL_SSH_INITD_SCRIPT@#$(SSH_INITD_SCRIPT)#g'; \
-         echo 's#@''PKGLIBDIR@#$(pkglibdir)#g'; \
+         echo 's#@''PKGLIBDIR@#$(libdir)/ganeti#g'; \
          echo 's#@''GNTMASTERUSER@#$(MASTERD_USER)#g'; \
          echo 's#@''GNTRAPIUSER@#$(RAPI_USER)#g'; \
          echo 's#@''GNTCONFDUSER@#$(CONFD_USER)#g'; \
+         echo 's#@''GNTLUXIDUSER@#$(LUXID_USER)#g'; \
          echo 's#@''GNTNODEDUSER@#$(NODED_USER)#g'; \
          echo 's#@''GNTMONDUSER@#$(MOND_USER)#g'; \
          echo 's#@''GNTRAPIGROUP@#$(RAPI_GROUP)#g'; \
          echo 's#@''GNTADMINGROUP@#$(ADMIN_GROUP)#g'; \
          echo 's#@''GNTCONFDGROUP@#$(CONFD_GROUP)#g'; \
+         echo 's#@''GNTLUXIDGROUP@#$(LUXID_GROUP)#g'; \
          echo 's#@''GNTMASTERDGROUP@#$(MASTERD_GROUP)#g'; \
          echo 's#@''GNTMONDGROUP@#$(MOND_GROUP)#g'; \
          echo 's#@''GNTDAEMONSGROUP@#$(DAEMONS_GROUP)#g'; \
@@ -1710,28 +1896,30 @@ check-local: check-dirs $(GENERATED_FILES)
        $(CHECK_VERSION) $(VERSION) $(top_srcdir)/NEWS
        RELEASE=$(PACKAGE_VERSION) $(CHECK_NEWS) < $(top_srcdir)/NEWS
        PYTHONPATH=. $(RUN_IN_TEMPDIR) $(CURDIR)/$(CHECK_IMPORTS) . $(standalone_python_modules)
-       @expver=$(VERSION_MAJOR).$(VERSION_MINOR); \
        error= ; \
-       if test "`head -n 1 $(top_srcdir)/README`" != "Ganeti $$expver"; then \
-         echo "Incorrect version in README, expected $$expver" >&2; \
-         error=1; \
-       fi; \
-       for file in doc/iallocator.rst doc/hooks.rst doc/virtual-cluster.rst \
-           doc/security.rst; do \
-         if test "`sed -ne '4 p' $(top_srcdir)/$$file`" != \
-           "Documents Ganeti version $$expver"; then \
-           echo "Incorrect version in $$file, expected $$expver" >&2; \
+       if [ "x`echo $(VERSION_SUFFIX)|grep 'alpha'`" == "x" ]; then \
+         expver=$(VERSION_MAJOR).$(VERSION_MINOR); \
+         if test "`head -n 1 $(top_srcdir)/README`" != "Ganeti $$expver"; then \
+           echo "Incorrect version in README, expected $$expver" >&2; \
+           error=1; \
+         fi; \
+         for file in doc/iallocator.rst doc/hooks.rst doc/virtual-cluster.rst \
+             doc/security.rst; do \
+           if test "`sed -ne '4 p' $(top_srcdir)/$$file`" != \
+             "Documents Ganeti version $$expver"; then \
+             echo "Incorrect version in $$file, expected $$expver" >&2; \
+             error=1; \
+           fi; \
+         done; \
+         if ! test -f $(top_srcdir)/doc/design-$$expver.rst; then \
+           echo "File $(top_srcdir)/doc/design-$$expver.rst not found" >&2; \
+           error=1; \
+         fi; \
+         if test "`sed -ne '5 p' $(top_srcdir)/doc/design-draft.rst`" != \
+           ".. Last updated for Ganeti $$expver"; then \
+           echo "doc/design-draft.rst was not updated for version $$expver" >&2; \
            error=1; \
          fi; \
-       done; \
-       if ! test -f $(top_srcdir)/doc/design-$$expver.rst; then \
-         echo "File $(top_srcdir)/doc/design-$$expver.rst not found" >&2; \
-         error=1; \
-       fi; \
-       if test "`sed -ne '5 p' $(top_srcdir)/doc/design-draft.rst`" != \
-         ".. Last updated for Ganeti $$expver"; then \
-         echo "doc/design-draft.rst was not updated for version $$expver" >&2; \
-         error=1; \
        fi; \
        for file in configure.ac $(HS_LIBTEST_SRCS) $(HS_PROG_SRCS); do \
          if test $$(wc --max-line-length < $(top_srcdir)/$$file) -gt 80; then \
@@ -1741,15 +1929,30 @@ check-local: check-dirs $(GENERATED_FILES)
        done; \
        test -z "$$error"
 
+.PHONY: hs-test-%
+hs-test-%: test/hs/htest | $(BUILT_PYTHON_SOURCES)
+       @rm -f htest.tix
+       test/hs/htest -t $*
+
 .PHONY: hs-tests
 hs-tests: test/hs/htest | $(BUILT_PYTHON_SOURCES)
        @rm -f htest.tix
        ./test/hs/htest
 
+.PHONY: hs-shell-%
+hs-shell-%: test/hs/hpc-htools test/hs/hpc-mon-collector \
+            $(HS_BUILT_TEST_HELPERS)
+       @rm -f hpc-htools.tix hpc-mon-collector.tix
+       HBINARY="./test/hs/hpc-htools" \
+       SHELLTESTARGS=$(SHELLTESTARGS) \
+       ./test/hs/offline-test.sh $*
+
 .PHONY: hs-shell
 hs-shell: test/hs/hpc-htools test/hs/hpc-mon-collector $(HS_BUILT_TEST_HELPERS)
        @rm -f hpc-htools.tix hpc-mon-collector.tix
-       HBINARY="./test/hs/hpc-htools" ./test/hs/offline-test.sh
+       HBINARY="./test/hs/hpc-htools" \
+       SHELLTESTARGS=$(SHELLTESTARGS) \
+       ./test/hs/offline-test.sh
 
 .PHONY: hs-check
 hs-check: hs-tests hs-shell
@@ -1772,7 +1975,7 @@ PEP8_IGNORE = E111,E121,E125,E127,E261,E501
 # For excluding pep8 expects filenames only, not whole paths
 PEP8_EXCLUDE = $(subst $(space),$(comma),$(strip $(notdir $(BUILT_PYTHON_SOURCES))))
 
-LINT_TARGETS = pylint pylint-qa
+LINT_TARGETS = pylint pylint-qa pylint-test
 if HAS_PEP8
 LINT_TARGETS += pep8
 endif
@@ -1794,6 +1997,12 @@ pylint-qa: $(GENERATED_FILES)
        cd $(top_srcdir)/qa && \
          PYTHONPATH=$(abs_top_srcdir) $(PYLINT) $(LINT_OPTS) \
          --rcfile  ../pylintrc $(patsubst qa/%.py,%,$(qa_scripts))
+# FIXME: lint all test code, not just the newly added test support
+pylint-test: $(GENERATED_FILES)
+       @test -n "$(PYLINT)" || { echo 'pylint' not found during configure; exit 1; }
+       cd $(top_srcdir) && \
+               PYTHONPATH=.:./test/py $(PYLINT) $(LINT_OPTS) \
+               --rcfile=pylintrc-test  $(python_test_support)
 
 .PHONY: pep8
 pep8: $(GENERATED_FILES)
@@ -1882,6 +2091,29 @@ install-exec-local:
        @mkdir_p@ "$(DESTDIR)${localstatedir}/lib/ganeti" \
          "$(DESTDIR)${localstatedir}/log/ganeti" \
          "$(DESTDIR)${localstatedir}/run/ganeti"
+if INSTALL_SYMLINKS
+       for dir in $(SYMLINK_TARGET_DIRS); do \
+         @mkdir_p@  $(DESTDIR)$$dir; \
+       done
+       for prog in $(HS_HTOOLS_PROGS); do \
+         $(LN_S) $(bindir)/$$prog $(DESTDIR)$(exec_prefix)/bin/$$prog; \
+       done
+       $(LN_S) $(bindir)/htools $(DESTDIR)$(libdir)/ganeti/iallocators/hail
+       for prog in $(all_sbin_scripts); do \
+         $(LN_S) $(sbindir)/$$prog $(DESTDIR)$(exec_prefix)/sbin/$$prog; \
+       done
+       for link in $(SYMLINK_DIRS); do \
+         $(LN_S) $(versiondir)$$link $(DESTDIR)$$link; \
+       done
+       for prog in $(all_myexeclib_scripts); do \
+         $(LN_S) $(pkglibdir)/$$prog $(DESTDIR)$(libdir)/ganeti/$$prog; \
+       done
+       if ! test -n '$(ENABLE_MANPAGES)'; then \
+         for man in $(manfullpath); do \
+           $(LN_S) $(versiondir)$(datarootdir)/man/$$man $(DESTDIR)$(datarootdir)/man/$$man; \
+         done; \
+       fi
+endif
 
 .PHONY: apidoc
 if WANT_HSAPIDOC
@@ -1945,11 +2177,16 @@ TAGS: $(GENERATED_FILES)
          etags -l python -a -
 
 .PHONY: coverage
+
+COVERAGE_TESTS=
 if WANT_HTOOLS
-coverage: py-coverage hs-coverage
-else
-coverage: py-coverage
+COVERAGE_TESTS += hs-coverage
 endif
+if PY_UNIT
+COVERAGE_TESTS += py-coverage
+endif
+
+coverage: $(COVERAGE_TESTS)
 
 .PHONY: py-coverage
 py-coverage: $(GENERATED_FILES) $(python_tests)
@@ -1972,7 +2209,7 @@ hs-coverage: $(haskell_tests) test/hs/hpc-htools test/hs/hpc-mon-collector
        hpc sum --union $(HPCEXCL) \
          htest.tix hpc-htools.tix hpc-mon-collector.tix > coverage-hs.tix
        hpc markup --destdir=$(COVERAGE_HS_DIR) coverage-hs.tix
-       hpc report coverage-hs.tix
+       hpc report coverage-hs.tix | tee $(COVERAGE_HS_DIR)/report.txt
        $(LN_S) -f hpc_index.html $(COVERAGE_HS_DIR)/index.html
 
 # Special "kind-of-QA" target for htools, needs special setup (all
@@ -1991,7 +2228,12 @@ live-test: all
          --srcdir=.. $(HPCEXCL) ; \
        hpc report --srcdir=.. live-test $(HPCEXCL)
 
-commit-check: distcheck lint apidoc
+commit-check: autotools-check distcheck lint apidoc
+
+autotools-check:
+       TESTDATA_DIR=./test/data shelltest $(SHELLTESTARGS) \
+  $(abs_top_srcdir)/test/autotools/*-*.test \
+  -- --hide-successes
 
 .PHONY: gitignore-check
 gitignore-check: