doc/admin.rst \
doc/design-2.0.rst \
doc/design-2.1.rst \
+ doc/design-2.2.rst \
doc/devnotes.rst \
doc/glossary.rst \
doc/hooks.rst \
from ganeti import constants
from ganeti import errors
from ganeti import daemon
+from ganeti import utils
from ganeti import ssconf
(constants.SOCKET_DIR, constants.SOCKET_DIR_MODE),
]
daemon.GenericMain(constants.MASTERD, parser, dirs,
- CheckMasterd, ExecMasterd)
+ CheckMasterd, ExecMasterd,
+ multithreaded=True)
if __name__ == "__main__":
dirs = [(val, constants.RUN_DIRS_MODE) for val in constants.SUB_RUN_DIRS]
dirs.append((constants.LOG_OS_DIR, 0750))
dirs.append((constants.LOCK_DIR, 1777))
- daemon.GenericMain(constants.NODED, parser, dirs, None, ExecNoded)
+ daemon.GenericMain(constants.NODED, parser, dirs, None, ExecNoded,
+ default_ssl_cert=constants.SSL_CERT_FILE,
+ default_ssl_key=constants.SSL_CERT_FILE)
if __name__ == '__main__':
dirs = [(val, constants.RUN_DIRS_MODE) for val in constants.SUB_RUN_DIRS]
dirs.append((constants.LOG_OS_DIR, 0750))
- daemon.GenericMain(constants.RAPI, parser, dirs, CheckRapi, ExecRapi)
+ daemon.GenericMain(constants.RAPI, parser, dirs, CheckRapi, ExecRapi,
+ default_ssl_cert=constants.RAPI_CERT_FILE,
+ default_ssl_key=constants.RAPI_CERT_FILE)
if __name__ == "__main__":
--- /dev/null
+=================
+Ganeti 2.2 design
+=================
+
+This document describes the major changes in Ganeti 2.2 compared to
+the 2.1 version.
+
+The 2.2 version will be a relatively small release. Its main aim is to
+avoid changing too much of the core code, while addressing issues and
+adding new features and improvements over 2.1, in a timely fashion.
+
+.. contents:: :depth: 4
+
+Objective
+=========
+
+Background
+==========
+
+Overview
+========
+
+Detailed design
+===============
+
+As for 2.1 we divide the 2.2 design into three areas:
+
+- core changes, which affect the master daemon/job queue/locking or
+ all/most logical units
+- logical unit/feature changes
+- external interface changes (eg. command line, os api, hooks, ...)
+
+Core changes
+------------
+
+Remote procedure call timeouts
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Current state and shortcomings
+++++++++++++++++++++++++++++++
+
+The current RPC protocol used by Ganeti is based on HTTP. Every request
+consists of an HTTP PUT request (e.g. ``PUT /hooks_runner HTTP/1.0``)
+and doesn't return until the function called has returned. Parameters
+and return values are encoded using JSON.
+
+On the server side, ``ganeti-noded`` handles every incoming connection
+in a separate process by forking just after accepting the connection.
+This process exits after sending the response.
+
+There is one major problem with this design: Timeouts can not be used on
+a per-request basis. Neither client or server know how long it will
+take. Even if we might be able to group requests into different
+categories (e.g. fast and slow), this is not reliable.
+
+If a node has an issue or the network connection fails while a request
+is being handled, the master daemon can wait for a long time for the
+connection to time out (e.g. due to the operating system's underlying
+TCP keep-alive packets or timeouts). While the settings for keep-alive
+packets can be changed using Linux-specific socket options, we prefer to
+use application-level timeouts because these cover both machine down and
+unresponsive node daemon cases.
+
+Proposed changes
+++++++++++++++++
+
+RPC glossary
+^^^^^^^^^^^^
+
+Function call ID
+ Unique identifier returned by ``ganeti-noded`` after invoking a
+ function.
+Function process
+ Process started by ``ganeti-noded`` to call actual (backend) function.
+
+Protocol
+^^^^^^^^
+
+Initially we chose HTTP as our RPC protocol because there were existing
+libraries, which, unfortunately, turned out to miss important features
+(such as SSL certificate authentication) and we had to write our own.
+
+This proposal can easily be implemented using HTTP, though it would
+likely be more efficient and less complicated to use the LUXI protocol
+already used to communicate between client tools and the Ganeti master
+daemon. Switching to another protocol can occur at a later point. This
+proposal should be implemented using HTTP as its underlying protocol.
+
+The LUXI protocol currently contains two functions, ``WaitForJobChange``
+and ``AutoArchiveJobs``, which can take a longer time. They both support
+a parameter to specify the timeout. This timeout is usually chosen as
+roughly half of the socket timeout, guaranteeing a response before the
+socket times out. After the specified amount of time,
+``AutoArchiveJobs`` returns and reports the number of archived jobs.
+``WaitForJobChange`` returns and reports a timeout. In both cases, the
+functions can be called again.
+
+A similar model can be used for the inter-node RPC protocol. In some
+sense, the node daemon will implement a light variant of *"node daemon
+jobs"*. When the function call is sent, it specifies an initial timeout.
+If the function didn't finish within this timeout, a response is sent
+with a unique identifier, the function call ID. The client can then
+choose to wait for the function to finish again with a timeout.
+Inter-node RPC calls would no longer be blocking indefinitely and there
+would be an implicit ping-mechanism.
+
+Request handling
+^^^^^^^^^^^^^^^^
+
+To support the protocol changes described above, the way the node daemon
+handles request will have to change. Instead of forking and handling
+every connection in a separate process, there should be one child
+process per function call and the master process will handle the
+communication with clients and the function processes using asynchronous
+I/O.
+
+Function processes communicate with the parent process via stdio and
+possibly their exit status. Every function process has a unique
+identifier, though it shouldn't be the process ID only (PIDs can be
+recycled and are prone to race conditions for this use case). The
+proposed format is ``${ppid}:${cpid}:${time}:${random}``, where ``ppid``
+is the ``ganeti-noded`` PID, ``cpid`` the child's PID, ``time`` the
+current Unix timestamp with decimal places and ``random`` at least 16
+random bits.
+
+The following operations will be supported:
+
+``StartFunction(fn_name, fn_args, timeout)``
+ Starts a function specified by ``fn_name`` with arguments in
+ ``fn_args`` and waits up to ``timeout`` seconds for the function
+ to finish. Fire-and-forget calls can be made by specifying a timeout
+ of 0 seconds (e.g. for powercycling the node). Returns three values:
+ function call ID (if not finished), whether function finished (or
+ timeout) and the function's return value.
+``WaitForFunction(fnc_id, timeout)``
+ Waits up to ``timeout`` seconds for function call to finish. Return
+ value same as ``StartFunction``.
+
+In the future, ``StartFunction`` could support an additional parameter
+to specify after how long the function process should be aborted.
+
+Simplified timing diagram::
+
+ Master daemon Node daemon Function process
+ |
+ Call function
+ (timeout 10s) -----> Parse request and fork for ----> Start function
+ calling actual function, then |
+ wait up to 10s for function to |
+ finish |
+ | |
+ ... ...
+ | |
+ Examine return <---- | |
+ value and wait |
+ again -------------> Wait another 10s for function |
+ | |
+ ... ...
+ | |
+ Examine return <---- | |
+ value and wait |
+ again -------------> Wait another 10s for function |
+ | |
+ ... ...
+ | |
+ | Function ends,
+ Get return value and forward <-- process exits
+ Process return <---- it to caller
+ value and continue
+ |
+
+.. TODO: Convert diagram above to graphviz/dot graphic
+
+On process termination (e.g. after having been sent a ``SIGTERM`` or
+``SIGINT`` signal), ``ganeti-noded`` should send ``SIGTERM`` to all
+function processes and wait for all of them to terminate.
+
+
+Feature changes
+---------------
+
+External interface changes
+--------------------------
+
+.. vim: set textwidth=72 :
security.rst
design-2.0.rst
design-2.1.rst
+ design-2.2.rst
locking.rst
hooks.rst
iallocator.rst
else:
network_port = None
- ##if self.op.vnc_bind_address is None:
- ## self.op.vnc_bind_address = constants.VNC_DEFAULT_BIND_ADDRESS
-
# this is needed because os.path.join does not accept None arguments
if self.op.file_storage_dir is None:
string_file_storage_dir = ""
RAPI = "ganeti-rapi"
MASTERD = "ganeti-masterd"
-MULTITHREADED_DAEMONS = frozenset([MASTERD])
-
-DAEMONS_SSL = {
- # daemon-name: (default-cert-path, default-key-path)
- NODED: (SSL_CERT_FILE, SSL_CERT_FILE),
- RAPI: (RAPI_CERT_FILE, RAPI_CERT_FILE),
-}
-
DAEMONS_PORTS = {
# daemon-name: ("proto", "default-port")
NODED: ("tcp", 1811),
CONFD: ("udp", 1814),
RAPI: ("tcp", 5080),
-}
+ }
+
DEFAULT_NODED_PORT = DAEMONS_PORTS[NODED][1]
DEFAULT_CONFD_PORT = DAEMONS_PORTS[CONFD][1]
DEFAULT_RAPI_PORT = DAEMONS_PORTS[RAPI][1]
RAPI: LOG_DIR + "rapi-daemon.log",
MASTERD: LOG_DIR + "master-daemon.log",
}
+
LOG_OS_DIR = LOG_DIR + "os"
LOG_WATCHER = LOG_DIR + "watcher.log"
LOG_COMMANDS = LOG_DIR + "commands.log"
self._signal_wait.append(owner)
-def GenericMain(daemon_name, optionparser, dirs, check_fn, exec_fn):
+def GenericMain(daemon_name, optionparser, dirs, check_fn, exec_fn,
+ multithreaded=False,
+ default_ssl_cert=None, default_ssl_key=None):
"""Shared main function for daemons.
@type daemon_name: string
@type exec_fn: function which accepts (options, args)
@param exec_fn: function that's executed with the daemon's pid file held, and
runs the daemon itself.
+ @type multithreaded: bool
+ @param multithreaded: Whether the daemon uses threads
+ @type default_ssl_cert: string
+ @param default_ssl_cert: Default SSL certificate path
+ @type default_ssl_key: string
+ @param default_ssl_key: Default SSL key path
"""
optionparser.add_option("-f", "--foreground", dest="fork",
optionparser.add_option("-d", "--debug", dest="debug",
help="Enable some debug messages",
default=False, action="store_true")
+
if daemon_name in constants.DAEMONS_PORTS:
- # for networked daemons we also allow choosing the bind port and address.
- # by default we use the port provided by utils.GetDaemonPort, and bind to
- # 0.0.0.0 (which is represented by and empty bind address.
- port = utils.GetDaemonPort(daemon_name)
+ default_bind_address = "0.0.0.0"
+ default_port = utils.GetDaemonPort(daemon_name)
+
+ # For networked daemons we allow choosing the port and bind address
optionparser.add_option("-p", "--port", dest="port",
- help="Network port (%s default)." % port,
- default=port, type="int")
+ help="Network port (default: %s)" % default_port,
+ default=default_port, type="int")
optionparser.add_option("-b", "--bind", dest="bind_address",
- help="Bind address",
- default="", metavar="ADDRESS")
+ help=("Bind address (default: %s)" %
+ default_bind_address),
+ default=default_bind_address, metavar="ADDRESS")
- if daemon_name in constants.DAEMONS_SSL:
- default_cert, default_key = constants.DAEMONS_SSL[daemon_name]
+ if default_ssl_key is not None and default_ssl_cert is not None:
optionparser.add_option("--no-ssl", dest="ssl",
help="Do not secure HTTP protocol with SSL",
default=True, action="store_false")
optionparser.add_option("-K", "--ssl-key", dest="ssl_key",
- help="SSL key",
- default=default_key, type="string")
+ help=("SSL key path (default: %s)" %
+ default_ssl_key),
+ default=default_ssl_key, type="string",
+ metavar="SSL_KEY_PATH")
optionparser.add_option("-C", "--ssl-cert", dest="ssl_cert",
- help="SSL certificate",
- default=default_cert, type="string")
+ help=("SSL certificate path (default: %s)" %
+ default_ssl_cert),
+ default=default_ssl_cert, type="string",
+ metavar="SSL_CERT_PATH")
- multithread = utils.no_fork = daemon_name in constants.MULTITHREADED_DAEMONS
+ # Disable the use of fork(2) if the daemon uses threads
+ utils.no_fork = multithreaded
options, args = optionparser.parse_args()
- if hasattr(options, 'ssl') and options.ssl:
- if not (options.ssl_cert and options.ssl_key):
- print >> sys.stderr, "Need key and certificate to use ssl"
- sys.exit(constants.EXIT_FAILURE)
- for fname in (options.ssl_cert, options.ssl_key):
- if not os.path.isfile(fname):
- print >> sys.stderr, "Need ssl file %s to run" % fname
+ if getattr(options, "ssl", False):
+ ssl_paths = {
+ "certificate": options.ssl_cert,
+ "key": options.ssl_key,
+ }
+
+ for name, path in ssl_paths.iteritems():
+ if not os.path.isfile(path):
+ print >> sys.stderr, "SSL %s file '%s' was not found" % (name, path)
sys.exit(constants.EXIT_FAILURE)
+ # TODO: By initiating http.HttpSslParams here we would only read the files
+ # once and have a proper validation (isfile returns False on directories)
+ # at the same time.
+
if check_fn is not None:
check_fn(options, args)
utils.SetupLogging(logfile=constants.DAEMONS_LOGFILES[daemon_name],
debug=options.debug,
stderr_logging=not options.fork,
- multithreaded=multithread)
+ multithreaded=multithreaded)
logging.info("%s daemon startup", daemon_name)
exec_fn(options, args)
finally:
"""
return self.error_message_format % values
+
class HttpServer(http.HttpBase, asyncore.dispatcher):
"""Generic HTTP server class
projects / ganeti-local / commitdiff