From: Guido Trotter Date: Thu, 25 Jul 2013 13:40:32 +0000 (+0200) Subject: Merge branch 'stable-2.8' into stable-2.9 X-Git-Tag: v2.9.0beta1~45 X-Git-Url: https://code.grnet.gr/git/ganeti-local/commitdiff_plain/17cda747f443ab8910e098c1978e68d00354c6f6?hp=-c Merge branch 'stable-2.8' into stable-2.9 * stable-2.8: Update NEWS and version for Ganeti 2.7.1 Add hvparam to disable VNET_HDR on tap devices daemon-util: pass --oknodo at rotate_logs Add logrotate example daemon-util: provide rotate_logs and rotate_all_logs actions Fix permission errors for split users Make RAPI export device names and UUIDs Conflicts: lib/constants.py: trivial lib/rapi/rlib2.py: trivial Signed-off-by: Guido Trotter Reviewed-by: Thomas Thrainer --- 17cda747f443ab8910e098c1978e68d00354c6f6 diff --combined .gitignore index c593b79,be68d6f..0e9b955 --- a/.gitignore +++ b/.gitignore @@@ -68,6 -68,7 +68,7 @@@ /doc/examples/bash_completion-debug /doc/examples/ganeti.cron /doc/examples/ganeti.initd + /doc/examples/ganeti.logrotate /doc/examples/ganeti-kvm-poweroff.initd /doc/examples/ganeti-master-role.ocf /doc/examples/ganeti-node-role.ocf @@@ -104,8 -105,6 +105,8 @@@ /tools/ensure-dirs /tools/users-setup /tools/vcluster-setup +/tools/vif-ganeti +/tools/net-common /tools/node-cleanup /tools/node-daemon-setup /tools/prepare-node-join diff --combined Makefile.am index 6bfa4b8,8aed59b..c359728 --- a/Makefile.am +++ b/Makefile.am @@@ -20,9 -20,6 +20,9 @@@ strip_hsroot = $(patsubst src/%,%,$(pat # Use bash in order to be able to use pipefail SHELL=/bin/bash +# Enable colors in shelltest +SHELLTESTARGS = "-c" + ACLOCAL_AMFLAGS = -I autotools BUILD_BASH_COMPLETION = $(top_srcdir)/autotools/build-bash-completion RUN_IN_TEMPDIR = $(top_srcdir)/autotools/run-in-tempdir @@@ -45,7 -42,6 +45,7 @@@ SHELL_ENV_INIT = autotools/shell-env-in clientdir = $(pkgpythondir)/client cmdlibdir = $(pkgpythondir)/cmdlib hypervisordir = $(pkgpythondir)/hypervisor +storagedir = $(pkgpythondir)/storage httpdir = $(pkgpythondir)/http masterddir = $(pkgpythondir)/masterd confddir = $(pkgpythondir)/confd @@@ -66,6 -62,8 +66,6 @@@ myexeclibdir = $(pkglibdir HS_DIRS = \ src \ src/Ganeti \ - src/Ganeti/Block \ - src/Ganeti/Block/Drbd \ src/Ganeti/Confd \ src/Ganeti/Curl \ src/Ganeti/DataCollectors \ @@@ -76,17 -74,11 +76,17 @@@ src/Ganeti/Hypervisor/Xen \ src/Ganeti/Monitoring \ src/Ganeti/Query \ + src/Ganeti/Storage \ + src/Ganeti/Storage/Diskstats \ + src/Ganeti/Storage/Drbd \ + src/Ganeti/Storage/Lvm \ test/hs \ test/hs/Test \ test/hs/Test/Ganeti \ - test/hs/Test/Ganeti/Block \ - test/hs/Test/Ganeti/Block/Drbd \ + test/hs/Test/Ganeti/Storage \ + test/hs/Test/Ganeti/Storage/Diskstats \ + test/hs/Test/Ganeti/Storage/Drbd \ + test/hs/Test/Ganeti/Storage/Lvm \ test/hs/Test/Ganeti/Confd \ test/hs/Test/Ganeti/HTools \ test/hs/Test/Ganeti/HTools/Backend \ @@@ -123,7 -115,6 +123,7 @@@ DIRS = lib/masterd \ lib/rapi \ lib/server \ + lib/storage \ lib/tools \ lib/utils \ lib/watcher \ @@@ -202,8 -193,6 +202,8 @@@ CLEANFILES = $(man_MANS) \ $(manhtml) \ tools/kvm-ifup \ + tools/vif-ganeti \ + tools/net-common \ tools/users-setup \ tools/vcluster-setup \ stamp-directories \ @@@ -255,6 -244,7 +255,7 @@@ BUILT_EXAMPLES = doc/examples/ganeti-kvm-poweroff.initd \ doc/examples/ganeti.cron \ doc/examples/ganeti.initd \ + doc/examples/ganeti.logrotate \ doc/examples/ganeti-master-role.ocf \ doc/examples/ganeti-node-role.ocf \ doc/examples/gnt-config-backup \ @@@ -272,6 -262,7 +273,6 @@@ pkgpython_PYTHON = lib/__init__.py \ lib/asyncnotifier.py \ lib/backend.py \ - lib/bdev.py \ lib/bootstrap.py \ lib/cli.py \ lib/compat.py \ @@@ -300,6 -291,7 +301,6 @@@ lib/serializer.py \ lib/ssconf.py \ lib/ssh.py \ - lib/storage.py \ lib/uidpool.py \ lib/vcluster.py \ lib/network.py \ @@@ -348,16 -340,6 +349,16 @@@ hypervisor_PYTHON = lib/hypervisor/hv_lxc.py \ lib/hypervisor/hv_xen.py +storage_PYTHON = \ + lib/storage/__init__.py \ + lib/storage/bdev.py \ + lib/storage/base.py \ + lib/storage/container.py \ + lib/storage/drbd.py \ + lib/storage/drbd_info.py \ + lib/storage/drbd_cmdgen.py \ + lib/storage/filestorage.py + rapi_PYTHON = \ lib/rapi/__init__.py \ lib/rapi/baserlib.py \ @@@ -416,7 -398,6 +417,7 @@@ utils_PYTHON = lib/utils/nodesetup.py \ lib/utils/process.py \ lib/utils/retry.py \ + lib/utils/storage.py \ lib/utils/text.py \ lib/utils/wrapper.py \ lib/utils/x509.py @@@ -444,7 -425,6 +445,7 @@@ docinput = doc/design-htools-2.3.rst \ doc/design-http-server.rst \ doc/design-impexp2.rst \ + doc/design-internal-shutdown.rst \ doc/design-lu-generated-jobs.rst \ doc/design-linuxha.rst \ doc/design-multi-reloc.rst \ @@@ -543,11 -523,12 +544,11 @@@ HPCEXCL = --exclude Main $(patsubst src.%,--exclude Test.%,$(subst /,.,$(patsubst %.hs,%, $(HS_LIB_SRCS)))) HS_LIB_SRCS = \ - src/Ganeti/Block/Drbd/Types.hs \ - src/Ganeti/Block/Drbd/Parser.hs \ src/Ganeti/BasicTypes.hs \ src/Ganeti/Common.hs \ src/Ganeti/Compat.hs \ src/Ganeti/Confd/Client.hs \ + src/Ganeti/Confd/ClientFunctions.hs \ src/Ganeti/Confd/Server.hs \ src/Ganeti/Confd/Types.hs \ src/Ganeti/Confd/Utils.hs \ @@@ -556,11 -537,7 +557,11 @@@ src/Ganeti/Curl/Multi.hs \ src/Ganeti/Daemon.hs \ src/Ganeti/DataCollectors/CLI.hs \ + src/Ganeti/DataCollectors/Diskstats.hs \ src/Ganeti/DataCollectors/Drbd.hs \ + src/Ganeti/DataCollectors/InstStatus.hs \ + src/Ganeti/DataCollectors/InstStatusTypes.hs \ + src/Ganeti/DataCollectors/Lv.hs \ src/Ganeti/DataCollectors/Program.hs \ src/Ganeti/DataCollectors/Types.hs \ src/Ganeti/Errors.hs \ @@@ -590,7 -567,6 +591,7 @@@ src/Ganeti/HTools/Program/Hroller.hs \ src/Ganeti/HTools/Program/Main.hs \ src/Ganeti/HTools/Types.hs \ + src/Ganeti/Hypervisor/Xen.hs \ src/Ganeti/Hypervisor/Xen/XmParser.hs \ src/Ganeti/Hypervisor/Xen/Types.hs \ src/Ganeti/Hash.hs \ @@@ -605,7 -581,6 +606,7 @@@ src/Ganeti/OpCodes.hs \ src/Ganeti/OpParams.hs \ src/Ganeti/Path.hs \ + src/Ganeti/Query/Cluster.hs \ src/Ganeti/Query/Common.hs \ src/Ganeti/Query/Export.hs \ src/Ganeti/Query/Filter.hs \ @@@ -620,13 -595,6 +621,13 @@@ src/Ganeti/Rpc.hs \ src/Ganeti/Runtime.hs \ src/Ganeti/Ssconf.hs \ + src/Ganeti/Storage/Diskstats/Parser.hs \ + src/Ganeti/Storage/Diskstats/Types.hs \ + src/Ganeti/Storage/Drbd/Parser.hs \ + src/Ganeti/Storage/Drbd/Types.hs \ + src/Ganeti/Storage/Lvm/LVParser.hs \ + src/Ganeti/Storage/Lvm/Types.hs \ + src/Ganeti/Storage/Utils.hs \ src/Ganeti/THH.hs \ src/Ganeti/Types.hs \ src/Ganeti/Utils.hs @@@ -634,6 -602,8 +635,6 @@@ HS_TEST_SRCS = \ test/hs/Test/Ganeti/Attoparsec.hs \ test/hs/Test/Ganeti/BasicTypes.hs \ - test/hs/Test/Ganeti/Block/Drbd/Parser.hs \ - test/hs/Test/Ganeti/Block/Drbd/Types.hs \ test/hs/Test/Ganeti/Common.hs \ test/hs/Test/Ganeti/Confd/Types.hs \ test/hs/Test/Ganeti/Confd/Utils.hs \ @@@ -665,10 -635,6 +666,10 @@@ test/hs/Test/Ganeti/Rpc.hs \ test/hs/Test/Ganeti/Runtime.hs \ test/hs/Test/Ganeti/Ssconf.hs \ + test/hs/Test/Ganeti/Storage/Diskstats/Parser.hs \ + test/hs/Test/Ganeti/Storage/Drbd/Parser.hs \ + test/hs/Test/Ganeti/Storage/Drbd/Types.hs \ + test/hs/Test/Ganeti/Storage/Lvm/LVParser.hs \ test/hs/Test/Ganeti/THH.hs \ test/hs/Test/Ganeti/TestCommon.hs \ test/hs/Test/Ganeti/TestHTools.hs \ @@@ -826,9 -792,7 +827,9 @@@ qa_scripts = qa/qa_error.py \ qa/qa_group.py \ qa/qa_instance.py \ + qa/qa_instance_utils.py \ qa/qa_job.py \ + qa/qa_monitoring.py \ qa/qa_node.py \ qa/qa_os.py \ qa/qa_rapi.py \ @@@ -957,8 -921,6 +958,8 @@@ nodist_pkglib_python_scripts = myexeclib_SCRIPTS = \ daemons/daemon-util \ tools/kvm-ifup \ + tools/vif-ganeti \ + tools/net-common \ $(pkglib_python_scripts) \ $(HS_MYEXECLIB_PROGS) @@@ -994,8 -956,6 +995,8 @@@ EXTRA_DIST = devel/upload \ devel/webserver \ tools/kvm-ifup.in \ + tools/vif-ganeti.in \ + tools/net-common.in \ tools/users-setup.in \ tools/vcluster-setup.in \ $(docinput) \ @@@ -1078,7 -1038,6 +1079,7 @@@ TEST_FILES = test/data/htools/hail-alloc-invalid-network.json \ test/data/htools/hail-alloc-invalid-twodisks.json \ test/data/htools/hail-alloc-restricted-network.json \ + test/data/htools/hail-alloc-spindles.json \ test/data/htools/hail-alloc-twodisks.json \ test/data/htools/hail-change-group.json \ test/data/htools/hail-invalid-reloc.json \ @@@ -1086,27 -1045,18 +1087,27 @@@ test/data/htools/hail-reloc-drbd.json \ test/data/htools/hbal-excl-tags.data \ test/data/htools/hbal-split-insts.data \ + test/data/htools/hspace-tiered-dualspec-exclusive.data \ test/data/htools/hspace-tiered-dualspec.data \ + test/data/htools/hspace-tiered-exclusive.data \ test/data/htools/hspace-tiered-ipolicy.data \ + test/data/htools/hspace-tiered-mixed.data \ test/data/htools/hspace-tiered-resourcetypes.data \ + test/data/htools/hspace-tiered-vcpu.data \ test/data/htools/hspace-tiered.data \ test/data/htools/invalid-node.data \ test/data/htools/missing-resources.data \ test/data/htools/multiple-master.data \ + test/data/htools/multiple-tags.data \ test/data/htools/n1-failure.data \ test/data/htools/rapi/groups.json \ test/data/htools/rapi/info.json \ test/data/htools/rapi/instances.json \ test/data/htools/rapi/nodes.json \ + test/data/htools/hroller-full.data \ + test/data/htools/hroller-nodegroups.data \ + test/data/htools/hroller-nonredundant.data \ + test/data/htools/hroller-online.data \ test/data/htools/unique-reboot-order.data \ test/hs/shelltests/htools-balancing.test \ test/hs/shelltests/htools-basic.test \ @@@ -1124,8 -1074,6 +1125,8 @@@ test/hs/shelltests/htools-mon-collector.test \ test/data/bdev-drbd-8.0.txt \ test/data/bdev-drbd-8.3.txt \ + test/data/bdev-drbd-8.4.txt \ + test/data/bdev-drbd-8.4-no-disk-params.txt \ test/data/bdev-drbd-disk.txt \ test/data/bdev-drbd-net-ip4.txt \ test/data/bdev-drbd-net-ip6.txt \ @@@ -1144,9 -1092,8 +1145,9 @@@ test/data/cert1.pem \ test/data/cert2.pem \ test/data/cluster_config_2.7.json \ - test/data/cluster_config_downgraded_2.7.json \ + test/data/cluster_config_2.8.json \ test/data/instance-minor-pairing.txt \ + test/data/instance-prim-sec.txt \ test/data/ip-addr-show-dummy0.txt \ test/data/ip-addr-show-lo-ipv4.txt \ test/data/ip-addr-show-lo-ipv6.txt \ @@@ -1160,7 -1107,6 +1161,7 @@@ test/data/kvm_0.9.1_help_boot_test.txt \ test/data/kvm_1.0_help.txt \ test/data/kvm_1.1.2_help.txt \ + test/data/lvs_lv.txt \ test/data/NEWS_OK.txt \ test/data/NEWS_previous_unreleased.txt \ test/data/ovfdata/compr_disk.vmdk.gz \ @@@ -1188,16 -1134,12 +1189,16 @@@ test/data/ovfdata/wrong_manifest.ovf \ test/data/ovfdata/wrong_ova.ova \ test/data/ovfdata/wrong_xml.ovf \ + test/data/proc_diskstats.txt \ test/data/proc_drbd8.txt \ test/data/proc_drbd80-emptyline.txt \ + test/data/proc_drbd80-emptyversion.txt \ test/data/proc_drbd83.txt \ test/data/proc_drbd83_sync.txt \ test/data/proc_drbd83_sync_want.txt \ test/data/proc_drbd83_sync_krnl2.6.39.txt \ + test/data/proc_drbd84.txt \ + test/data/proc_drbd84_sync.txt \ test/data/qa-minimal-nodes-instances-only.json \ test/data/sys_drbd_usermode_helper.txt \ test/data/vgreduce-removemissing-2.02.02.txt \ @@@ -1222,14 -1164,12 +1223,14 @@@ python_tests = test/py/ganeti.asyncnotifier_unittest.py \ test/py/ganeti.backend_unittest-runasroot.py \ test/py/ganeti.backend_unittest.py \ - test/py/ganeti.bdev_unittest.py \ + test/py/ganeti.bootstrap_unittest.py \ test/py/ganeti.cli_unittest.py \ test/py/ganeti.client.gnt_cluster_unittest.py \ test/py/ganeti.client.gnt_instance_unittest.py \ test/py/ganeti.client.gnt_job_unittest.py \ test/py/ganeti.cmdlib_unittest.py \ + test/py/ganeti.cmdlib.cluster_unittest.py \ + test/py/ganeti.cmdlib.instance_storage_unittest.py \ test/py/ganeti.compat_unittest.py \ test/py/ganeti.confd.client_unittest.py \ test/py/ganeti.config_unittest.py \ @@@ -1271,10 -1211,7 +1272,10 @@@ test/py/ganeti.server.rapi_unittest.py \ test/py/ganeti.ssconf_unittest.py \ test/py/ganeti.ssh_unittest.py \ - test/py/ganeti.storage_unittest.py \ + test/py/ganeti.storage.bdev_unittest.py \ + test/py/ganeti.storage.container_unittest.py \ + test/py/ganeti.storage.drbd_unittest.py \ + test/py/ganeti.storage.filestorage_unittest.py \ test/py/ganeti.tools.burnin_unittest.py \ test/py/ganeti.tools.ensure_dirs_unittest.py \ test/py/ganeti.tools.node_daemon_setup_unittest.py \ @@@ -1291,7 -1228,6 +1292,7 @@@ test/py/ganeti.utils.nodesetup_unittest.py \ test/py/ganeti.utils.process_unittest.py \ test/py/ganeti.utils.retry_unittest.py \ + test/py/ganeti.utils.storage_unittest.py \ test/py/ganeti.utils.text_unittest.py \ test/py/ganeti.utils.wrapper_unittest.py \ test/py/ganeti.utils.x509_unittest.py \ @@@ -1351,7 -1287,6 +1352,7 @@@ all_python_code = $(client_PYTHON) \ $(cmdlib_PYTHON) \ $(hypervisor_PYTHON) \ + $(storage_PYTHON) \ $(rapi_PYTHON) \ $(server_PYTHON) \ $(pytools_PYTHON) \ @@@ -1427,14 -1362,6 +1428,14 @@@ tools/kvm-ifup: tools/kvm-ifup.in $(REP sed -f $(REPLACE_VARS_SED) < $< > $@ chmod +x $@ +tools/vif-ganeti: tools/vif-ganeti.in $(REPLACE_VARS_SED) + sed -f $(REPLACE_VARS_SED) < $< > $@ + chmod +x $@ + +tools/net-common: tools/net-common.in $(REPLACE_VARS_SED) + sed -f $(REPLACE_VARS_SED) < $< > $@ + chmod +x $@ + tools/users-setup: tools/users-setup.in $(REPLACE_VARS_SED) sed -f $(REPLACE_VARS_SED) < $< > $@ chmod +x $@ @@@ -1587,6 -1514,8 +1588,6 @@@ lib/_autoconf.py: Makefile | stamp-dire echo "XEN_KERNEL = '$(XEN_KERNEL)'"; \ echo "XEN_INITRD = '$(XEN_INITRD)'"; \ echo "KVM_KERNEL = '$(KVM_KERNEL)'"; \ - echo "FILE_STORAGE_DIR = '$(FILE_STORAGE_DIR)'"; \ - echo "ENABLE_FILE_STORAGE = $(ENABLE_FILE_STORAGE)"; \ echo "SHARED_FILE_STORAGE_DIR = '$(SHARED_FILE_STORAGE_DIR)'"; \ echo "ENABLE_SHARED_FILE_STORAGE = $(ENABLE_SHARED_FILE_STORAGE)"; \ echo "IALLOCATOR_SEARCH_PATH = [$(IALLOCATOR_SEARCH_PATH)]"; \ @@@ -1688,6 -1617,7 +1689,6 @@@ $(REPLACE_VARS_SED): $(SHELL_ENV_INIT) echo 's#@''CUSTOM_XEN_INITRD@#$(XEN_INITRD)#g'; \ echo 's#@''CUSTOM_IALLOCATOR_SEARCH_PATH@#$(IALLOCATOR_SEARCH_PATH)#g'; \ echo 's#@''CUSTOM_EXPORT_DIR@#$(EXPORT_DIR)#g'; \ - echo 's#@''RPL_FILE_STORAGE_DIR@#$(FILE_STORAGE_DIR)#g'; \ echo 's#@''RPL_SSH_INITD_SCRIPT@#$(SSH_INITD_SCRIPT)#g'; \ echo 's#@''PKGLIBDIR@#$(pkglibdir)#g'; \ echo 's#@''GNTMASTERUSER@#$(MASTERD_USER)#g'; \ @@@ -1843,30 -1773,15 +1844,30 @@@ check-local: check-dirs $(GENERATED_FIL done; \ test -z "$$error" +.PHONY: hs-test-% +hs-test-%: test/hs/htest | $(BUILT_PYTHON_SOURCES) + @rm -f htest.tix + test/hs/htest -t $* + .PHONY: hs-tests hs-tests: test/hs/htest | $(BUILT_PYTHON_SOURCES) @rm -f htest.tix ./test/hs/htest +.PHONY: hs-shell-% +hs-shell-%: test/hs/hpc-htools test/hs/hpc-mon-collector \ + $(HS_BUILT_TEST_HELPERS) + @rm -f hpc-htools.tix hpc-mon-collector.tix + HBINARY="./test/hs/hpc-htools" \ + SHELLTESTARGS=$(SHELLTESTARGS) \ + ./test/hs/offline-test.sh $* + .PHONY: hs-shell hs-shell: test/hs/hpc-htools test/hs/hpc-mon-collector $(HS_BUILT_TEST_HELPERS) @rm -f hpc-htools.tix hpc-mon-collector.tix - HBINARY="./test/hs/hpc-htools" ./test/hs/offline-test.sh + HBINARY="./test/hs/hpc-htools" \ + SHELLTESTARGS=$(SHELLTESTARGS) \ + ./test/hs/offline-test.sh .PHONY: hs-check hs-check: hs-tests hs-shell diff --combined NEWS index cb7ef77,30c348a..7f9766f --- a/NEWS +++ b/NEWS @@@ -2,48 -2,6 +2,48 @@@ New ==== +Version 2.9.0 beta1 +------------------- + +*(unreleased)* + +- DRBD 8.4 support. Depending on the installed DRBD version, Ganeti now uses + the correct command syntax. It is possible to use different DRBD versions + on different nodes as long as they are compatible to each other. This + enables rolling upgrades of DRBD with no downtime. As permanent operation + of different DRBD versions within a node group is discouraged, + ``gnt-cluster verify`` will emit a warning if it detects such a situation. +- hroller now also plans for capacity to move non-redundant instances off + any node to be rebooted; the old behavior of completely ignoring any + non-redundant instances can be restored by adding the --ignore-non-redundant + option. +- The cluster option '--no-lvm-storage' was removed in favor of the new option + '--enabled-disk-templates'. +- On instance creation, disk templates no longer need to be specified + with '-t'. The default disk template will be taken from the list of + enabled disk templates. +- New "inst-status-xen" data collector for the monitoring daemon, providing + information about the state of the xen instances on the nodes. +- New "lv" data collector for the monitoring daemon, collecting data about the + logical volumes on the nodes, and pairing them with the name of the instances + they belong to. +- New "diskstats" data collector, collecting the data from /proc/diskstats and + presenting them over the monitoring daemon interface. +- The monitoring daemon is now running as root, in order to be able to collect + information only available to root (such as the state of Xen instances). +- The ConfD client is now IPv6 compatible. +- File storage is no longer dis/enabled at configure time, but using the + option '--enabled-disk-templates' at cluster initialization and + modification. +- The default directory for file storage is not anymore specified at + configure time, but taken from the cluster's configuration. It can be + set at cluster initialization and modification with '--file-storage-dir'. +- Cluster verification now includes stricter checks regarding the + default file storage dir. It now checks that the file storage + directory is explicitely allowed in the 'file-storage-paths' file and + that the directory exists on all nodes. + + Version 2.8.0 beta1 ------------------- @@@ -116,6 -74,20 +116,20 @@@ For Python - ``yaml`` library (only for running the QA). + Version 2.7.1 + ------------- + + *(Released Thu, 25 Jul 2013)* + + - Add logrotate functionality in daemon-util + - Add logrotate example file + - Add missing fields to network queries over rapi + - Fix network object timestamps + - Add support for querying network timestamps + - Fix a typo in the example crontab + - Fix a documentation typo + + Version 2.7.0 ------------- diff --combined doc/install.rst index d797111,3be446c..04ea7c0 --- a/doc/install.rst +++ b/doc/install.rst @@@ -498,21 -498,6 +498,21 @@@ commands: bridge name bridge id STP enabled interfaces xen-br0 8000.0020fc1ed55d no eth0 +In order to have a custom and more advanced networking configuration in Xen +which can vary among instances, after having successfully installed Ganeti +you have to create a symbolic link to the vif-script provided by Ganeti +inside /etc/xen/scripts (assuming you installed Ganeti under /usr/lib):: + + $ ln -s /usr/lib/ganeti/vif-ganeti /etc/xen/scripts/vif-ganeti + +This has to be done on all nodes. Afterwards you can set the ``vif_script`` +hypervisor parameter to point to that script by:: + + $ gnt-cluster modify -H xen-pvm:vif_script=/etc/xen/scripts/vif-ganeti + +Having this hypervisor parameter you are able to create your own scripts +and create instances with different networking configurations. + .. _configure-lvm-label: Configuring LVM @@@ -582,7 -567,10 +582,10 @@@ distribution's startup scripts, for exa In order to automatically restart failed instances, you need to setup a cron job run the *ganeti-watcher* command. A sample cron file is provided in the source at ``doc/examples/ganeti.cron`` and you can copy - that (eventually altering the path) to ``/etc/cron.d/ganeti``. + that (eventually altering the path) to ``/etc/cron.d/ganeti``. Finally, + a sample logrotate snippet is provided in the source at + ``doc/examples/ganeti.logrotate`` and you can copy it to + ``/etc/logrotate.d/ganeti`` to have Ganeti's logs rotated automatically. What gets installed ~~~~~~~~~~~~~~~~~~~ diff --combined lib/constants.py index fe4f88d,339ead8..0e721ba --- a/lib/constants.py +++ b/lib/constants.py @@@ -145,6 -145,7 +145,6 @@@ RUN_DIRS_MODE = 077 SECURE_DIR_MODE = 0700 SECURE_FILE_MODE = 0600 ADOPTABLE_BLOCKDEV_ROOT = "/dev/disk/" -ENABLE_FILE_STORAGE = _autoconf.ENABLE_FILE_STORAGE ENABLE_SHARED_FILE_STORAGE = _autoconf.ENABLE_SHARED_FILE_STORAGE ENABLE_CONFD = _autoconf.ENABLE_CONFD ENABLE_MOND = _autoconf.ENABLE_MOND @@@ -191,7 -192,7 +191,7 @@@ LAST_DRBD_PORT = 1499 DAEMONS_LOGBASE = { NODED: "node-daemon", CONFD: "conf-daemon", - LUXID: "query-daemon", + LUXID: "luxi-daemon", RAPI: "rapi-daemon", MASTERD: "master-daemon", MOND: "monitoring-daemon", @@@ -254,6 -255,8 +254,6 @@@ XEN_KERNEL = _autoconf.XEN_KERNE XEN_INITRD = _autoconf.XEN_INITRD XEN_CMD_XM = "xm" XEN_CMD_XL = "xl" -# FIXME: This will be made configurable using hvparams in Ganeti 2.7 -XEN_CMD = _autoconf.XEN_CMD KNOWN_XEN_COMMANDS = compat.UniqueFrozenset([ XEN_CMD_XM, @@@ -384,7 -387,7 +384,7 @@@ ST_LVM_PV = "lvm-pv ST_LVM_VG = "lvm-vg" ST_RADOS = "rados" -VALID_STORAGE_TYPES = compat.UniqueFrozenset([ +STORAGE_TYPES = compat.UniqueFrozenset([ ST_BLOCK, ST_DISKLESS, ST_EXT, @@@ -394,10 -397,6 +394,10 @@@ ST_RADOS, ]) +# the set of storage types for which storage reporting is available +# FIXME: Remove this, once storage reporting is available for all types. +STS_REPORT = compat.UniqueFrozenset([ST_FILE, ST_LVM_PV, ST_LVM_VG]) + # Storage fields # first two are valid in LU context only, not passed to backend SF_NODE = "node" @@@ -436,12 -435,6 +436,12 @@@ VALID_STORAGE_OPERATIONS = LDS_UNKNOWN, LDS_FAULTY) = range(1, 4) +LDS_NAMES = { + LDS_OKAY: "ok", + LDS_UNKNOWN: "unknown", + LDS_FAULTY: "faulty", +} + # disk template types DT_BLOCK = "blockdev" DT_DISKLESS = "diskless" @@@ -457,14 -450,14 +457,14 @@@ DT_SHARED_FILE = "sharedfile # This only happens on an upgrade from a version of Ganeti that did not # support the 'enabled_disk_templates' so far. DISK_TEMPLATE_PREFERENCE = [ + DT_BLOCK, + DT_DISKLESS, DT_DRBD8, - DT_PLAIN, + DT_EXT, DT_FILE, - DT_SHARED_FILE, + DT_PLAIN, DT_RBD, - DT_BLOCK, - DT_DISKLESS, - DT_EXT + DT_SHARED_FILE, ] DISK_TEMPLATES = compat.UniqueFrozenset([ @@@ -479,13 -472,13 +479,13 @@@ ]) # disk templates that are enabled by default -DEFAULT_ENABLED_DISK_TEMPLATES = compat.UniqueFrozenset([ +DEFAULT_ENABLED_DISK_TEMPLATES = [ DT_DRBD8, DT_PLAIN, - ]) + ] # mapping of disk templates to storage types -DISK_TEMPLATES_STORAGE_TYPE = { +MAP_DISK_TEMPLATE_STORAGE_TYPE = { DT_BLOCK: ST_BLOCK, DT_DISKLESS: ST_DISKLESS, DT_DRBD8: ST_LVM_VG, @@@ -591,8 -584,7 +591,8 @@@ LDS_BLOCK = compat.UniqueFrozenset( # drbd constants DRBD_HMAC_ALG = "md5" -DRBD_NET_PROTOCOL = "C" +DRBD_DEFAULT_NET_PROTOCOL = "C" +DRBD_MIGRATION_NET_PROTOCOL = "C" DRBD_STATUS_FILE = "/proc/drbd" #: Size of DRBD meta block device @@@ -957,8 -949,7 +957,9 @@@ HV_KVM_EXTRA = "kvm_extra HV_KVM_MACHINE_VERSION = "machine_version" HV_KVM_PATH = "kvm_path" HV_VIF_TYPE = "vif_type" +HV_VIF_SCRIPT = "vif_script" +HV_XEN_CMD = "xen_cmd" + HV_VNET_HDR = "vnet_hdr" HVS_PARAMETER_TYPES = { @@@ -1029,8 -1020,7 +1030,9 @@@ HV_KVM_EXTRA: VTYPE_STRING, HV_KVM_MACHINE_VERSION: VTYPE_STRING, HV_VIF_TYPE: VTYPE_STRING, + HV_VIF_SCRIPT: VTYPE_STRING, + HV_XEN_CMD: VTYPE_STRING, + HV_VNET_HDR: VTYPE_BOOL, } HVS_PARAMETERS = frozenset(HVS_PARAMETER_TYPES.keys()) @@@ -1217,7 -1207,6 +1219,7 @@@ LDP_NO_META_FLUSH = "disable-meta-flush LDP_DEFAULT_METAVG = "default-metavg" LDP_DISK_CUSTOM = "disk-custom" LDP_NET_CUSTOM = "net-custom" +LDP_PROTOCOL = "protocol" LDP_DYNAMIC_RESYNC = "dynamic-resync" LDP_PLAN_AHEAD = "c-plan-ahead" LDP_FILL_TARGET = "c-fill-target" @@@ -1233,7 -1222,6 +1235,7 @@@ DISK_LD_TYPES = LDP_DEFAULT_METAVG: VTYPE_STRING, LDP_DISK_CUSTOM: VTYPE_STRING, LDP_NET_CUSTOM: VTYPE_STRING, + LDP_PROTOCOL: VTYPE_STRING, LDP_DYNAMIC_RESYNC: VTYPE_BOOL, LDP_PLAN_AHEAD: VTYPE_INT, LDP_FILL_TARGET: VTYPE_INT, @@@ -1254,7 -1242,6 +1256,7 @@@ DRBD_META_BARRIERS = "meta-barriers DRBD_DEFAULT_METAVG = "metavg" DRBD_DISK_CUSTOM = "disk-custom" DRBD_NET_CUSTOM = "net-custom" +DRBD_PROTOCOL = "protocol" DRBD_DYNAMIC_RESYNC = "dynamic-resync" DRBD_PLAN_AHEAD = "c-plan-ahead" DRBD_FILL_TARGET = "c-fill-target" @@@ -1272,7 -1259,6 +1274,7 @@@ DISK_DT_TYPES = DRBD_DEFAULT_METAVG: VTYPE_STRING, DRBD_DISK_CUSTOM: VTYPE_STRING, DRBD_NET_CUSTOM: VTYPE_STRING, + DRBD_PROTOCOL: VTYPE_STRING, DRBD_DYNAMIC_RESYNC: VTYPE_BOOL, DRBD_PLAN_AHEAD: VTYPE_INT, DRBD_FILL_TARGET: VTYPE_INT, @@@ -1347,7 -1333,6 +1349,7 @@@ NICS_PARAMETERS = frozenset(NICS_PARAME # IDISK_* constants are used in opcodes, to create/change disks IDISK_SIZE = "size" +IDISK_SPINDLES = "spindles" IDISK_MODE = "mode" IDISK_ADOPT = "adopt" IDISK_VG = "vg" @@@ -1356,7 -1341,6 +1358,7 @@@ IDISK_PROVIDER = "provider IDISK_NAME = "name" IDISK_PARAMS_TYPES = { IDISK_SIZE: VTYPE_SIZE, + IDISK_SPINDLES: VTYPE_INT, IDISK_MODE: VTYPE_STRING, IDISK_ADOPT: VTYPE_STRING, IDISK_VG: VTYPE_STRING, @@@ -1610,13 -1594,8 +1612,13 @@@ CV_EINSTANCEPOLICY = CV_EINSTANCEUNSUITABLENODE = \ (CV_TINSTANCE, "EINSTANCEUNSUITABLENODE", "Instance running on nodes that are not suitable for it") +CV_EINSTANCEMISSINGCFGPARAMETER = \ + (CV_TINSTANCE, "EINSTANCEMISSINGCFGPARAMETER", + "A configuration parameter for an instance is missing") CV_ENODEDRBD = \ (CV_TNODE, "ENODEDRBD", "Error parsing the DRBD status file") +CV_ENODEDRBDVERSION = \ + (CV_TNODE, "ENODEDRBDVERSION", "DRBD version mismatch within a node group") CV_ENODEDRBDHELPER = \ (CV_TNODE, "ENODEDRBDHELPER", "Error caused by the DRBD helper") CV_ENODEFILECHECK = \ @@@ -1656,8 -1635,6 +1658,8 @@@ CV_ENODEUSERSCRIPTS = (CV_TNODE, "ENODEUSERSCRIPTS", "User scripts not present or not executable") CV_ENODEFILESTORAGEPATHS = \ (CV_TNODE, "ENODEFILESTORAGEPATHS", "Detected bad file storage paths") +CV_ENODEFILESTORAGEPATHUNUSABLE = \ + (CV_TNODE, "ENODEFILESTORAGEPATHUNUSABLE", "File storage path unusable") CV_ALL_ECODES = compat.UniqueFrozenset([ CV_ECLUSTERCFG, @@@ -1692,7 -1669,6 +1694,7 @@@ CV_ENODEOOBPATH, CV_ENODEUSERSCRIPTS, CV_ENODEFILESTORAGEPATHS, + CV_ENODEFILESTORAGEPATHUNUSABLE, ]) CV_ALL_ECODES_STRINGS = \ @@@ -1701,13 -1677,10 +1703,13 @@@ # Node verify constants NV_BRIDGES = "bridges" NV_DRBDHELPER = "drbd-helper" +NV_DRBDVERSION = "drbd-version" NV_DRBDLIST = "drbd-list" NV_EXCLUSIVEPVS = "exclusive-pvs" NV_FILELIST = "filelist" -NV_FILE_STORAGE_PATHS = "file-storage-paths" +NV_ACCEPTED_STORAGE_PATHS = "allowed-file-storage-paths" +NV_FILE_STORAGE_PATH = "file-storage-path" +NV_SHARED_FILE_STORAGE_PATH = "shared-file-storage-path" NV_HVINFO = "hvinfo" NV_HVPARAMS = "hvparms" NV_HYPERVISOR = "hypervisor" @@@ -2031,26 -2004,6 +2033,26 @@@ SS_UID_POOL = "uid_pool SS_NODEGROUPS = "nodegroups" SS_NETWORKS = "networks" +# This is not a complete SSCONF key, but the prefix for the hypervisor keys +SS_HVPARAMS_PREF = "hvparams_" + +# Hvparams keys: +SS_HVPARAMS_XEN_PVM = SS_HVPARAMS_PREF + HT_XEN_PVM +SS_HVPARAMS_XEN_FAKE = SS_HVPARAMS_PREF + HT_FAKE +SS_HVPARAMS_XEN_HVM = SS_HVPARAMS_PREF + HT_XEN_HVM +SS_HVPARAMS_XEN_KVM = SS_HVPARAMS_PREF + HT_KVM +SS_HVPARAMS_XEN_CHROOT = SS_HVPARAMS_PREF + HT_CHROOT +SS_HVPARAMS_XEN_LXC = SS_HVPARAMS_PREF + HT_LXC + +VALID_SS_HVPARAMS_KEYS = compat.UniqueFrozenset([ + SS_HVPARAMS_XEN_PVM, + SS_HVPARAMS_XEN_FAKE, + SS_HVPARAMS_XEN_HVM, + SS_HVPARAMS_XEN_KVM, + SS_HVPARAMS_XEN_CHROOT, + SS_HVPARAMS_XEN_LXC, + ]) + SS_FILE_PERMS = 0444 # cluster wide default parameters @@@ -2072,8 -2025,6 +2074,8 @@@ HVC_DEFAULTS = HV_CPU_MASK: CPU_PINNING_ALL, HV_CPU_CAP: 0, HV_CPU_WEIGHT: 256, + HV_VIF_SCRIPT: "", + HV_XEN_CMD: XEN_CMD_XM, }, HT_XEN_HVM: { HV_BOOT_ORDER: "cd", @@@ -2096,8 -2047,6 +2098,8 @@@ HV_CPU_CAP: 0, HV_CPU_WEIGHT: 256, HV_VIF_TYPE: HT_HVM_VIF_IOEMU, + HV_VIF_SCRIPT: "", + HV_XEN_CMD: XEN_CMD_XM, }, HT_KVM: { HV_KVM_PATH: KVM_PATH, @@@ -2156,10 -2105,9 +2158,11 @@@ HV_VGA: "", HV_KVM_EXTRA: "", HV_KVM_MACHINE_VERSION: "", + HV_VNET_HDR: True, }, - HT_FAKE: {}, + HT_FAKE: { + HV_MIGRATION_MODE: HT_MIGRATION_LIVE, + }, HT_CHROOT: { HV_INIT_SCRIPT: "/ganeti-chroot", }, @@@ -2172,7 -2120,6 +2175,7 @@@ HVC_GLOBALS = compat.UniqueFrozenset( HV_MIGRATION_PORT, HV_MIGRATION_BANDWIDTH, HV_MIGRATION_MODE, + HV_XEN_CMD, ]) BEC_DEFAULTS = { @@@ -2202,7 -2149,6 +2205,7 @@@ DISK_LD_DEFAULTS = LDP_DEFAULT_METAVG: DEFAULT_VG, LDP_DISK_CUSTOM: "", LDP_NET_CUSTOM: "", + LDP_PROTOCOL: DRBD_DEFAULT_NET_PROTOCOL, LDP_DYNAMIC_RESYNC: False, # The default values for the DRBD dynamic resync speed algorithm @@@ -2244,7 -2190,6 +2247,7 @@@ DISK_DT_DEFAULTS = DRBD_DEFAULT_METAVG: _DRBD_DEFAULTS[LDP_DEFAULT_METAVG], DRBD_DISK_CUSTOM: _DRBD_DEFAULTS[LDP_DISK_CUSTOM], DRBD_NET_CUSTOM: _DRBD_DEFAULTS[LDP_NET_CUSTOM], + DRBD_PROTOCOL: _DRBD_DEFAULTS[LDP_PROTOCOL], DRBD_DYNAMIC_RESYNC: _DRBD_DEFAULTS[LDP_DYNAMIC_RESYNC], DRBD_PLAN_AHEAD: _DRBD_DEFAULTS[LDP_PLAN_AHEAD], DRBD_FILL_TARGET: _DRBD_DEFAULTS[LDP_FILL_TARGET], @@@ -2529,7 -2474,5 +2532,7 @@@ OPCODE_REASON_SOURCES = compat.UniqueFr OPCODE_REASON_SRC_USER, ]) +DISKSTATS_FILE = "/proc/diskstats" + # Do not re-export imported modules del re, _vcsversion, _autoconf, socket, pathutils, compat diff --combined lib/hypervisor/hv_kvm.py index 4bd8a9e,b3eaf7e..14c5807 --- a/lib/hypervisor/hv_kvm.py +++ b/lib/hypervisor/hv_kvm.py @@@ -460,7 -460,11 +460,7 @@@ class KVMHypervisor(hv_base.BaseHypervi constants.HV_ACPI: hv_base.NO_CHECK, constants.HV_SERIAL_CONSOLE: hv_base.NO_CHECK, constants.HV_SERIAL_SPEED: hv_base.NO_CHECK, - constants.HV_VNC_BIND_ADDRESS: - (False, lambda x: (netutils.IP4Address.IsValid(x) or - utils.IsNormAbsPath(x)), - "The VNC bind address must be either a valid IP address or an absolute" - " pathname", None, None), + constants.HV_VNC_BIND_ADDRESS: hv_base.NO_CHECK, # will be checked later constants.HV_VNC_TLS: hv_base.NO_CHECK, constants.HV_VNC_X509: hv_base.OPT_DIR_CHECK, constants.HV_VNC_X509_VERIFY: hv_base.NO_CHECK, @@@ -529,6 -533,7 +529,7 @@@ constants.HV_VGA: hv_base.NO_CHECK, constants.HV_KVM_EXTRA: hv_base.NO_CHECK, constants.HV_KVM_MACHINE_VERSION: hv_base.NO_CHECK, + constants.HV_VNET_HDR: hv_base.NO_CHECK, } _VIRTIO = "virtio" @@@ -567,7 -572,6 +568,7 @@@ # dashes not preceeded by a new line (which would mean another option # different than -drive is starting) _BOOT_RE = re.compile(r"^-drive\s([^-]|(? constants.VNC_BASE_PORT: display = instance.network_port - constants.VNC_BASE_PORT @@@ -1395,10 -1387,6 +1396,10 @@@ for dev in hvp[constants.HV_USB_DEVICES].split(","): kvm_cmd.extend(["-usbdevice", dev]) + # Set system UUID to instance UUID + if self._UUID_RE.search(kvmhelp): + kvm_cmd.extend(["-uuid", instance.uuid]) + if hvp[constants.HV_KVM_EXTRA]: kvm_cmd.extend(hvp[constants.HV_KVM_EXTRA].split(" ")) @@@ -1533,7 -1521,7 +1534,7 @@@ devlist = self._GetKVMOutput(kvm_path, self._KVMOPT_DEVICELIST) if self._NEW_VIRTIO_RE.search(devlist): nic_model = self._VIRTIO_NET_PCI - vnet_hdr = True + vnet_hdr = up_hvp[constants.HV_VNET_HDR] except errors.HypervisorError, _: # Older versions of kvm don't support DEVICE_LIST, but they don't # have new virtio syntax either. @@@ -1894,14 -1882,12 +1895,14 @@@ else: self.StopInstance(instance, force=True) - def MigrateInstance(self, instance, target, live): + def MigrateInstance(self, cluster_name, instance, target, live): """Migrate an instance to a target node. The migration will not be attempted if the instance is not currently running. + @type cluster_name: string + @param cluster_name: name of the cluster @type instance: L{objects.Instance} @param instance: the instance to be migrated @type target: string @@@ -1997,14 -1983,13 +1998,14 @@@ """ self._CallMonitorCommand(instance.name, "balloon %d" % mem) - def GetNodeInfo(self): + def GetNodeInfo(self, hvparams=None): """Return information about the node. - @return: a dict with the following keys (values in MiB): - - memory_total: the total memory size on the node - - memory_free: the available memory on the node for instances - - memory_dom0: the memory used by the node itself, if available + @type hvparams: dict of strings + @param hvparams: hypervisor parameters, not used in this class + + @return: a dict as returned by L{BaseHypervisor.GetLinuxNodeInfo} plus + the following keys: - hv_version: the hypervisor version in the form (major, minor, revision) @@@ -2018,7 -2003,7 +2019,7 @@@ return result @classmethod - def GetInstanceConsole(cls, instance, hvparams, beparams): + def GetInstanceConsole(cls, instance, primary_node, hvparams, beparams): """Return a command for connecting to the console of an instance. """ @@@ -2030,7 -2015,7 +2031,7 @@@ "UNIX-CONNECT:%s" % cls._InstanceSerial(instance.name)] return objects.InstanceConsole(instance=instance.name, kind=constants.CONS_SSH, - host=instance.primary_node, + host=primary_node.name, user=constants.SSH_CONSOLE_USER, command=cmd) @@@ -2055,14 -2040,11 +2056,14 @@@ message=("No serial shell for instance %s" % instance.name)) - def Verify(self): + def Verify(self, hvparams=None): """Verify the hypervisor. Check that the required binaries exist. + @type hvparams: dict of strings + @param hvparams: hypervisor parameters to be verified against, not used here + @return: Problem description if something is wrong, C{None} otherwise """ @@@ -2172,16 -2154,6 +2173,16 @@@ except KeyError: raise errors.HypervisorError("Unknown security domain user %s" % username) + vnc_bind_address = hvparams[constants.HV_VNC_BIND_ADDRESS] + if vnc_bind_address: + bound_to_addr = netutils.IP4Address.IsValid(vnc_bind_address) + is_interface = netutils.IsValidInterface(vnc_bind_address) + is_path = utils.IsNormAbsPath(vnc_bind_address) + if not bound_to_addr and not is_interface and not is_path: + raise errors.HypervisorError("VNC: The %s parameter must be either" + " a valid IP address, an interface name," + " or an absolute path" % + constants.HV_KVM_SPICE_BIND) spice_bind = hvparams[constants.HV_KVM_SPICE_BIND] if spice_bind: @@@ -2213,11 -2185,8 +2214,11 @@@ machine_version) @classmethod - def PowercycleNode(cls): + def PowercycleNode(cls, hvparams=None): """KVM powercycle, just a wrapper over Linux powercycle. + @type hvparams: dict of strings + @param hvparams: hypervisor params to be used on this node + """ cls.LinuxPowercycle() diff --combined lib/rapi/rlib2.py index 8fef531,321ddb5..d8d9b8c --- a/lib/rapi/rlib2.py +++ b/lib/rapi/rlib2.py @@@ -71,20 -71,20 +71,21 @@@ _COMMON_FIELDS = ["ctime", "mtime", "uu I_FIELDS = ["name", "admin_state", "os", "pnode", "snodes", "disk_template", - "nic.ips", "nic.macs", "nic.modes", + "nic.ips", "nic.macs", "nic.modes", "nic.uuids", "nic.names", "nic.links", "nic.networks", "nic.networks.names", "nic.bridges", "network_port", - "disk.sizes", "disk.spindles", "disk_usage", - "disk.sizes", "disk_usage", "disk.uuids", "disk.names", ++ "disk.sizes", "disk.spindles", "disk_usage", "disk.uuids", ++ "disk.names", "beparams", "hvparams", "oper_state", "oper_ram", "oper_vcpus", "status", "custom_hvparams", "custom_beparams", "custom_nicparams", ] + _COMMON_FIELDS N_FIELDS = ["name", "offline", "master_candidate", "drained", - "dtotal", "dfree", + "dtotal", "dfree", "sptotal", "spfree", "mtotal", "mnode", "mfree", "pinst_cnt", "sinst_cnt", - "ctotal", "cnodes", "csockets", + "ctotal", "cnos", "cnodes", "csockets", "pip", "sip", "role", "pinst_list", "sinst_list", "master_capable", "vm_capable",