X-Git-Url: https://code.grnet.gr/git/ganeti-local/blobdiff_plain/9ca87fb31173b18119fb39c18dc3668e78a4054a..db04ce5de18bc05eff45e8535399f91249a75a8c:/Makefile.am diff --git a/Makefile.am b/Makefile.am index 3ec9b22..147ab6c 100644 --- a/Makefile.am +++ b/Makefile.am @@ -9,6 +9,11 @@ abs_top_builddir = @abs_top_builddir@ abs_top_srcdir = @abs_top_srcdir@ +# Helper values for calling builtin functions +empty := +space := $(empty) $(empty) +comma := , + # Use bash in order to be able to use pipefail SHELL=/bin/bash @@ -19,9 +24,11 @@ CHECK_PYTHON_CODE = $(top_srcdir)/autotools/check-python-code CHECK_MAN = $(top_srcdir)/autotools/check-man CHECK_VERSION = $(top_srcdir)/autotools/check-version CHECK_NEWS = $(top_srcdir)/autotools/check-news +CHECK_IMPORTS = $(top_srcdir)/autotools/check-imports DOCPP = $(top_srcdir)/autotools/docpp REPLACE_VARS_SED = autotools/replace_vars.sed CONVERT_CONSTANTS = $(top_srcdir)/autotools/convert-constants +BUILD_RPC = $(top_srcdir)/autotools/build-rpc # Note: these are automake-specific variables, and must be named after # the directory + 'dir' suffix @@ -75,6 +82,8 @@ DIRS = \ qa \ test \ test/data \ + test/data/ovfdata \ + test/data/ovfdata/other \ tools BUILDTIME_DIR_AUTOCREATE = \ @@ -136,6 +145,7 @@ CLEANFILES = \ doc/examples/ganeti.cron \ doc/examples/gnt-config-backup \ doc/examples/hooks/ipsec \ + lib/_generated_rpc.py \ $(man_MANS) \ $(manhtml) \ tools/kvm-ifup \ @@ -150,17 +160,21 @@ CLEANFILES = \ BUILT_SOURCES = \ ganeti \ stamp-srclinks \ + $(all_dirfiles) \ + $(PYTHON_BOOTSTRAP) \ + $(BUILT_PYTHON_SOURCES) + +BUILT_PYTHON_SOURCES = \ lib/_autoconf.py \ lib/_vcsversion.py \ - $(all_dirfiles) \ - $(PYTHON_BOOTSTRAP) + lib/_generated_rpc.py nodist_pkgpython_PYTHON = \ - lib/_autoconf.py \ - lib/_vcsversion.py + $(BUILT_PYTHON_SOURCES) noinst_PYTHON = \ lib/build/__init__.py \ + lib/build/rpc_definitions.py \ lib/build/sphinx_ext.py pkgpython_PYTHON = \ @@ -185,6 +199,7 @@ pkgpython_PYTHON = \ lib/netutils.py \ lib/objects.py \ lib/opcodes.py \ + lib/ovf.py \ lib/qlang.py \ lib/query.py \ lib/rpc.py \ @@ -245,7 +260,8 @@ impexpd_PYTHON = \ watcher_PYTHON = \ lib/watcher/__init__.py \ - lib/watcher/nodemaint.py + lib/watcher/nodemaint.py \ + lib/watcher/state.py server_PYTHON = \ lib/server/__init__.py \ @@ -281,6 +297,7 @@ docrst = \ doc/design-2.3.rst \ doc/design-htools-2.3.rst \ doc/design-2.4.rst \ + doc/design-2.5.rst \ doc/design-draft.rst \ doc/design-oob.rst \ doc/design-cpu-pinning.rst \ @@ -292,8 +309,12 @@ docrst = \ doc/design-multi-reloc.rst \ doc/design-network.rst \ doc/design-chained-jobs.rst \ + doc/design-ovf-support.rst \ + doc/design-resource-model.rst \ doc/cluster-merge.rst \ doc/design-shared-storage.rst \ + doc/design-node-state-cache.rst \ + doc/design-virtual-clusters.rst \ doc/devnotes.rst \ doc/glossary.rst \ doc/hooks.rst \ @@ -319,8 +340,11 @@ HFLAGS = -O -Wall -fwarn-monomorphism-restriction -fwarn-tabs -ihtools # extra flags that can be overriden on the command line HEXTRA = # exclude options for coverage reports -HPCEXCL = --exclude Main --exclude Ganeti.HTools.QC \ +HPCEXCL = --exclude Main \ --exclude Ganeti.Constants \ + --exclude Ganeti.THH \ + --exclude Ganeti.HTools.QC \ + --exclude Ganeti.HTools.QCHelper \ --exclude Ganeti.HTools.Version HS_LIB_SRCS = \ @@ -332,11 +356,13 @@ HS_LIB_SRCS = \ htools/Ganeti/HTools/Group.hs \ htools/Ganeti/HTools/IAlloc.hs \ htools/Ganeti/HTools/Instance.hs \ + htools/Ganeti/HTools/JSON.hs \ htools/Ganeti/HTools/Loader.hs \ htools/Ganeti/HTools/Luxi.hs \ htools/Ganeti/HTools/Node.hs \ htools/Ganeti/HTools/PeerMap.hs \ htools/Ganeti/HTools/QC.hs \ + htools/Ganeti/HTools/QCHelper.hs \ htools/Ganeti/HTools/Rapi.hs \ htools/Ganeti/HTools/Simu.hs \ htools/Ganeti/HTools/Text.hs \ @@ -348,7 +374,8 @@ HS_LIB_SRCS = \ htools/Ganeti/HTools/Program/Hspace.hs \ htools/Ganeti/Jobs.hs \ htools/Ganeti/Luxi.hs \ - htools/Ganeti/OpCodes.hs + htools/Ganeti/OpCodes.hs \ + htools/Ganeti/THH.hs HS_BUILT_SRCS = htools/Ganeti/HTools/Version.hs htools/Ganeti/Constants.hs HS_BUILT_SRCS_IN = $(patsubst %,%.in,$(HS_BUILT_SRCS)) @@ -362,7 +389,7 @@ $(RUN_IN_TEMPDIR): | $(all_dirfiles) # it changes doc/html/index.html: $(docrst) $(docpng) doc/conf.py configure.ac \ $(RUN_IN_TEMPDIR) lib/build/sphinx_ext.py lib/opcodes.py lib/ht.py \ - | lib/_autoconf.py lib/_vcsversion.py + | $(BUILT_PYTHON_SOURCES) @test -n "$(SPHINX)" || \ { echo 'sphinx-build' not found during configure; exit 1; } @mkdir_p@ $(dir $@) @@ -444,6 +471,7 @@ qa_scripts = \ qa/qa_daemon.py \ qa/qa_env.py \ qa/qa_error.py \ + qa/qa_group.py \ qa/qa_instance.py \ qa/qa_node.py \ qa/qa_os.py \ @@ -456,10 +484,11 @@ if WANT_HTOOLS bin_SCRIPTS += $(filter-out htools/hail,$(HS_PROGS)) install-exec-hook: @mkdir_p@ $(DESTDIR)$(iallocatorsdir) - $(LN_S) -f $(DESTDIR)$(bindir)/htools \ +# FIXME: this is a hardcoded logic, instead of auto-resolving + $(LN_S) -f ../../../bin/htools \ $(DESTDIR)$(iallocatorsdir)/hail for role in $(HS_BIN_ROLES); do \ - $(LN_S) -f $(DESTDIR)$(bindir)/htools \ + $(LN_S) -f htools \ $(DESTDIR)$(bindir)/$$role ; \ done endif @@ -475,9 +504,10 @@ $(HS_ALL_PROGS): %: %.hs $(HS_LIB_SRCS) $(HS_BUILT_SRCS) Makefile exit 1; \ fi BINARY=$(@:htools/%=%); $(GHC) --make \ - $(HFLAGS) $(HEXTRA) \ + $(HFLAGS) \ $(HTOOLS_NOCURL) $(HTOOLS_PARALLEL3) \ - -osuf $$BINARY.o -hisuf $$BINARY.hi $@ + -osuf $$BINARY.o -hisuf $$BINARY.hi \ + $(HEXTRA) $@ # for the htools/test binary, we need to enable profiling/coverage htools/test: HEXTRA=-fhpc -Wwarn -fno-warn-missing-signatures \ @@ -499,6 +529,7 @@ dist_tools_PYTHON = \ tools/cluster-merge \ tools/lvmstrap \ tools/move-instance \ + tools/ovfconverter \ tools/setup-ssh \ tools/sanitize-config @@ -528,7 +559,9 @@ EXTRA_DIST = \ epydoc.conf.in \ pylintrc \ autotools/build-bash-completion \ + autotools/build-rpc \ autotools/check-python-code \ + autotools/check-imports \ autotools/check-man \ autotools/check-news \ autotools/check-tar \ @@ -610,10 +643,44 @@ TEST_FILES = \ test/data/bdev-drbd-net-ip4.txt \ test/data/bdev-drbd-net-ip6.txt \ test/data/cert1.pem \ + test/data/ip-addr-show-dummy0.txt \ + test/data/ip-addr-show-lo-ipv4.txt \ + test/data/ip-addr-show-lo-ipv6.txt \ + test/data/ip-addr-show-lo-oneline-ipv4.txt \ + test/data/ip-addr-show-lo-oneline-ipv6.txt \ + test/data/ip-addr-show-lo-oneline.txt \ + test/data/ip-addr-show-lo.txt \ test/data/proc_drbd8.txt \ test/data/proc_drbd80-emptyline.txt \ test/data/proc_drbd83.txt \ + test/data/proc_drbd83_sync.txt \ + test/data/proc_drbd83_sync_krnl2.6.39.txt \ test/data/sys_drbd_usermode_helper.txt \ + test/data/ovfdata/compr_disk.vmdk.gz \ + test/data/ovfdata/config.ini \ + test/data/ovfdata/corrupted_resources.ovf \ + test/data/ovfdata/empty.ini \ + test/data/ovfdata/empty.ovf \ + test/data/ovfdata/ganeti.mf \ + test/data/ovfdata/ganeti.ovf \ + test/data/ovfdata/gzip_disk.ovf \ + test/data/ovfdata/new_disk.vmdk \ + test/data/ovfdata/no_disk.ini \ + test/data/ovfdata/no_disk_in_ref.ovf \ + test/data/ovfdata/no_os.ini \ + test/data/ovfdata/no_ovf.ova \ + test/data/ovfdata/ova.ova \ + test/data/ovfdata/second_disk.vmdk \ + test/data/ovfdata/rawdisk.raw \ + test/data/ovfdata/unsafe_path.ini \ + test/data/ovfdata/virtualbox.ovf \ + test/data/ovfdata/wrong_extension.ovd \ + test/data/ovfdata/wrong_config.ini \ + test/data/ovfdata/wrong_manifest.mf \ + test/data/ovfdata/wrong_manifest.ovf \ + test/data/ovfdata/wrong_ova.ova \ + test/data/ovfdata/wrong_xml.ovf \ + test/data/ovfdata/other/rawdisk.raw \ test/import-export_unittest-helper python_tests = \ @@ -648,6 +715,7 @@ python_tests = \ test/ganeti.netutils_unittest.py \ test/ganeti.objects_unittest.py \ test/ganeti.opcodes_unittest.py \ + test/ganeti.ovf_unittest.py \ test/ganeti.qlang_unittest.py \ test/ganeti.query_unittest.py \ test/ganeti.rapi.baserlib_unittest.py \ @@ -676,6 +744,7 @@ python_tests = \ test/ganeti.workerpool_unittest.py \ test/cfgupgrade_unittest.py \ test/docs_unittest.py \ + test/pycurl_reset_unittest.py \ test/tempfile_fork_unittest.py haskell_tests = htools/test @@ -734,6 +803,7 @@ srclink_files = \ check_python_code = \ $(BUILD_BASH_COMPLETION) \ + $(CHECK_IMPORTS) \ $(DOCPP) \ $(all_python_code) @@ -744,9 +814,25 @@ lint_python_code = \ $(dist_tools_PYTHON) \ $(pkglib_python_scripts) \ $(BUILD_BASH_COMPLETION) \ + $(CHECK_IMPORTS) \ $(DOCPP) \ $(PYTHON_BOOTSTRAP) +standalone_python_modules = \ + lib/rapi/client.py \ + tools/ganeti-listrunner + +pep8_python_code = \ + ganeti \ + ganeti/http/server.py \ + $(dist_sbin_SCRIPTS) \ + $(dist_tools_PYTHON) \ + $(pkglib_python_scripts) \ + $(BUILD_BASH_COMPLETION) \ + $(DOCPP) \ + $(PYTHON_BOOTSTRAP) \ + qa + test/daemon-util_unittest.bash: daemons/daemon-util test/ganeti-cleaner_unittest.bash: daemons/ganeti-cleaner @@ -842,7 +928,8 @@ htools/Ganeti/HTools/Version.hs: htools/Ganeti/HTools/Version.hs.in vcs-version sed -e "s/%ver%/$$VCSVER/" < $< > $@ htools/Ganeti/Constants.hs: htools/Ganeti/Constants.hs.in \ - lib/constants.py lib/_autoconf.py $(CONVERT_CONSTANTS) + lib/constants.py lib/_autoconf.py $(CONVERT_CONSTANTS) \ + | lib/_vcsversion.py set -e; \ { cat $< ; PYTHONPATH=. $(CONVERT_CONSTANTS); } > $@ @@ -859,7 +946,7 @@ lib/_autoconf.py: Makefile | lib/.dir echo ''; \ echo '"""'; \ echo ''; \ - echo '# pylint: disable-msg=C0301,C0324'; \ + echo '# pylint: disable=C0301,C0324'; \ echo '# because this is autogenerated, we do not want'; \ echo '# style warnings' ; \ echo ''; \ @@ -883,6 +970,7 @@ lib/_autoconf.py: Makefile | lib/.dir echo "ENABLE_SHARED_FILE_STORAGE = $(ENABLE_SHARED_FILE_STORAGE)"; \ echo "IALLOCATOR_SEARCH_PATH = [$(IALLOCATOR_SEARCH_PATH)]"; \ echo "KVM_PATH = '$(KVM_PATH)'"; \ + echo "IP_PATH = '$(IP_PATH)'"; \ echo "SOCAT_PATH = '$(SOCAT)'"; \ echo "SOCAT_USE_ESCAPE = $(SOCAT_USE_ESCAPE)"; \ echo "SOCAT_USE_COMPRESS = $(SOCAT_USE_COMPRESS)"; \ @@ -903,6 +991,7 @@ lib/_autoconf.py: Makefile | lib/.dir echo "NODED_USER = '$(NODED_USER)'"; \ echo "NODED_GROUP = '$(NODED_GROUP)'"; \ echo "DISK_SEPARATOR = '$(DISK_SEPARATOR)'"; \ + echo "QEMUIMG_PATH = '$(QEMUIMG_PATH)'"; \ if [ "$(HTOOLS)" ]; then \ echo "HTOOLS = True"; \ else \ @@ -924,13 +1013,16 @@ lib/_vcsversion.py: Makefile vcs-version | lib/.dir echo ''; \ echo '"""'; \ echo ''; \ - echo '# pylint: disable-msg=C0301,C0324'; \ + echo '# pylint: disable=C0301,C0324'; \ echo '# because this is autogenerated, we do not want'; \ echo '# style warnings' ; \ echo ''; \ echo "VCS_VERSION = '$$VCSVER'"; \ } > $@ +lib/_generated_rpc.py: lib/build/rpc_definitions.py $(BUILD_RPC) | lib/.dir + PYTHONPATH=. $(RUN_IN_TEMPDIR) $(CURDIR)/$(BUILD_RPC) $< > $@ + $(REPLACE_VARS_SED): Makefile set -e; \ { echo 's#@PREFIX@#$(prefix)#g'; \ @@ -973,7 +1065,7 @@ $(PYTHON_BOOTSTRAP): Makefile | $(all_dirfiles) echo; \ echo '"""Bootstrap script for L{$(MODULE)}"""'; \ echo; \ - echo '# pylint: disable-msg=C0103'; \ + echo '# pylint: disable=C0103'; \ echo '# C0103: Invalid name'; \ echo; \ echo 'import sys'; \ @@ -982,9 +1074,9 @@ $(PYTHON_BOOTSTRAP): Makefile | $(all_dirfiles) echo '# Temporarily alias commands until bash completion'; \ echo '# generator is changed'; \ echo 'if hasattr(main, "commands"):'; \ - echo ' commands = main.commands # pylint: disable-msg=E1101'; \ + echo ' commands = main.commands # pylint: disable=E1101'; \ echo 'if hasattr(main, "aliases"):'; \ - echo ' aliases = main.aliases # pylint: disable-msg=E1101'; \ + echo ' aliases = main.aliases # pylint: disable=E1101'; \ echo; \ echo 'if __name__ == "__main__":'; \ echo ' sys.exit(main.Main())'; \ @@ -1029,29 +1121,47 @@ check-dirs: $(BUILT_SOURCES) if test -n "$$error"; then exit 1; else exit 0; fi; \ } -check-local: check-dirs +.PHONY: check-local +check-local: check-dirs $(BUILT_SOURCES) $(CHECK_PYTHON_CODE) $(check_python_code) $(CHECK_VERSION) $(VERSION) $(top_srcdir)/NEWS $(CHECK_NEWS) < $(top_srcdir)/NEWS + PYTHONPATH=. $(RUN_IN_TEMPDIR) $(CURDIR)/$(CHECK_IMPORTS) . $(standalone_python_modules) expver=$(VERSION_MAJOR).$(VERSION_MINOR); \ if test "`head -n 1 $(top_srcdir)/README`" != "Ganeti $$expver"; then \ echo "Incorrect version in README, expected $$expver"; \ exit 1; \ fi; \ - if test "`sed -ne '4 p' $(top_srcdir)/doc/iallocator.rst`" != \ - "Documents Ganeti version $$expver"; then \ - echo "Incorrect version in iallocator.rst, expected $$expver"; \ - exit 1; \ - fi + for file in doc/iallocator.rst doc/hooks.rst; do \ + if test "`sed -ne '4 p' $(top_srcdir)/$$file`" != \ + "Documents Ganeti version $$expver"; then \ + echo "Incorrect version in $$file, expected $$expver"; \ + exit 1; \ + fi; \ + done .PHONY: hs-check hs-check: htools/test @rm -f test.tix ./htools/test +# E111: indentation is not a multiple of four +# E261: at least two spaces before inline comment +# E501: line too long (80 characters) +PEP8_IGNORE = E111,E261,E501 + +# For excluding pep8 expects filenames only, not whole paths +PEP8_EXCLUDE = $(subst $(space),$(comma),$(strip $(notdir $(BUILT_PYTHON_SOURCES)))) + .PHONY: lint lint: $(BUILT_SOURCES) @test -n "$(PYLINT)" || { echo 'pylint' not found during configure; exit 1; } + if test -z "$(PEP8)"; then \ + echo '"pep8" not found during configure' >&2; \ + else \ + $(PEP8) --repeat --ignore='$(PEP8_IGNORE)' --exclude='$(PEP8_EXCLUDE)' \ + $(pep8_python_code); \ + fi $(PYLINT) $(LINT_OPTS) $(lint_python_code) cd $(top_srcdir)/qa && \ PYTHONPATH=$(abs_top_srcdir) $(PYLINT) $(LINT_OPTS) \ @@ -1142,7 +1252,7 @@ hs-apidoc: $(HS_BUILT_SRCS) rm -rf $(APIDOC_HS_DIR)/* @mkdir_p@ $(APIDOC_HS_DIR)/Ganeti/HTools/Program $(HSCOLOUR) -print-css > $(APIDOC_HS_DIR)/Ganeti/hscolour.css - ln -s ../hscolour.css $(APIDOC_HS_DIR)/Ganeti/HTools/hscolour.css + $(LN_S) ../hscolour.css $(APIDOC_HS_DIR)/Ganeti/HTools/hscolour.css set -e ; \ cd htools; \ if [ "$(HTOOLS_NOCURL)" ]; \ @@ -1194,7 +1304,7 @@ hs-coverage: $(haskell_tests) @mkdir_p@ $(COVERAGE_HS_DIR) hpc markup --destdir=$(COVERAGE_HS_DIR) htools/test $(HPCEXCL) hpc report htools/test $(HPCEXCL) - ln -sf hpc_index.html $(COVERAGE_HS_DIR)/index.html + $(LN_S) -f hpc_index.html $(COVERAGE_HS_DIR)/index.html # Special "kind-of-QA" target for htools, needs special setup (all # tools compiled with -fhpc) @@ -1202,7 +1312,7 @@ hs-coverage: $(haskell_tests) live-test: all set -e ; \ cd htools; \ - rm -f .hpc; ln -s ../.hpc .hpc; \ + rm -f .hpc; $(LN_S) ../.hpc .hpc; \ rm -f *.tix *.mix; \ ./live-test.sh; \ hpc sum --union $(HPCEXCL) $(addsuffix .tix,$(HS_PROGS:htools/%=%)) \