Merge branch 'devel-2.1'

* devel-2.1:
  utils: Add class to split string stream into lines
  Fix cluster behaviour with disabled file storage
  Update docstrings in tools/ and enable epydoc
  Forward-port the ganeti 2.0 cfgupgrade
  Add a new tool: sanitize-config
  Fix cfgupgrade with non-default DATA_DIR
  Improving the RAPI documentation
  Mark cluster-merge as executable
  QA: Make the rapi credential handling less involving

Conflicts:
	lib/bootstrap.py (merge cds and new file names)
	lib/cmdlib.py    (trivial, kept 2.1 version for disabled file
	                  storage)
	lib/utils.py     (trivial, new imports)
	tools/cfgupgrade (trivial, new cds parameter)

Signed-off-by: Iustin Pop <iustin@google.com>
Reviewed-by: Michael Hanselmann <hansmi@google.com>

diff --cc Makefile.am
Simple merge

diff --cc lib/bootstrap.py
index 3eec3f6,f686598..fe094b3
--- 1/lib/bootstrap.py
--- 2/lib/bootstrap.py
+++ b/lib/bootstrap.py
@@@ -77,7 -112,10 +77,10 @@@ def GenerateHmacKey(file_name)
  
  
  def GenerateClusterCrypto(new_cluster_cert, new_rapi_cert, new_confd_hmac_key,
-                           new_cds, rapi_cert_pem=None, cds=None):
 -                          rapi_cert_pem=None,
++                          new_cds, rapi_cert_pem=None, cds=None,
+                           nodecert_file=constants.NODED_CERT_FILE,
+                           rapicert_file=constants.RAPI_CERT_FILE,
+                           hmackey_file=constants.CONFD_HMAC_KEY):
    """Updates the cluster certificates, keys and secrets.
  
    @type new_cluster_cert: bool
@@@ -86,31 -124,32 +89,36 @@@
    @param new_rapi_cert: Whether to generate a new RAPI certificate
    @type new_confd_hmac_key: bool
    @param new_confd_hmac_key: Whether to generate a new HMAC key
 +  @type new_cds: bool
 +  @param new_cds: Whether to generate a new cluster domain secret
    @type rapi_cert_pem: string
    @param rapi_cert_pem: New RAPI certificate in PEM format
 +  @type cds: string
 +  @param cds: New cluster domain secret
+   @type nodecert_file: string
+   @param nodecert_file: optional override of the node cert file path
+   @type rapicert_file: string
+   @param rapicert_file: optional override of the rapi cert file path
+   @type hmackey_file: string
+   @param hmackey_file: optional override of the hmac key file path
  
    """
    # noded SSL certificate
-   cluster_cert_exists = os.path.exists(constants.NODED_CERT_FILE)
+   cluster_cert_exists = os.path.exists(nodecert_file)
    if new_cluster_cert or not cluster_cert_exists:
      if cluster_cert_exists:
-       utils.CreateBackup(constants.NODED_CERT_FILE)
+       utils.CreateBackup(nodecert_file)
  
-     logging.debug("Generating new cluster certificate at %s",
-                   constants.NODED_CERT_FILE)
-     utils.GenerateSelfSignedSslCert(constants.NODED_CERT_FILE)
+     logging.debug("Generating new cluster certificate at %s", nodecert_file)
 -    GenerateSelfSignedSslCert(nodecert_file)
++    utils.GenerateSelfSignedSslCert(nodecert_file)
  
    # confd HMAC key
-   if new_confd_hmac_key or not os.path.exists(constants.CONFD_HMAC_KEY):
-     logging.debug("Writing new confd HMAC key to %s", constants.CONFD_HMAC_KEY)
-     GenerateHmacKey(constants.CONFD_HMAC_KEY)
+   if new_confd_hmac_key or not os.path.exists(hmackey_file):
+     logging.debug("Writing new confd HMAC key to %s", hmackey_file)
+     GenerateHmacKey(hmackey_file)
  
    # RAPI
-   rapi_cert_exists = os.path.exists(constants.RAPI_CERT_FILE)
+   rapi_cert_exists = os.path.exists(rapicert_file)
  
    if rapi_cert_pem:
      # Assume rapi_pem contains a valid PEM-formatted certificate and key
@@@ -120,23 -158,10 +127,22 @@@
  
    elif new_rapi_cert or not rapi_cert_exists:
      if rapi_cert_exists:
-       utils.CreateBackup(constants.RAPI_CERT_FILE)
+       utils.CreateBackup(rapicert_file)
  
-     logging.debug("Generating new RAPI certificate at %s",
-                   constants.RAPI_CERT_FILE)
-     utils.GenerateSelfSignedSslCert(constants.RAPI_CERT_FILE)
+     logging.debug("Generating new RAPI certificate at %s", rapicert_file)
 -    GenerateSelfSignedSslCert(rapicert_file)
++    utils.GenerateSelfSignedSslCert(rapicert_file)
 +
 +  # Cluster domain secret
 +  if cds:
 +    logging.debug("Writing cluster domain secret to %s",
 +                  constants.CLUSTER_DOMAIN_SECRET_FILE)
 +    utils.WriteFile(constants.CLUSTER_DOMAIN_SECRET_FILE,
 +                    data=cds, backup=True)
 +
 +  elif new_cds or not os.path.exists(constants.CLUSTER_DOMAIN_SECRET_FILE):
 +    logging.debug("Generating new cluster domain secret at %s",
 +                  constants.CLUSTER_DOMAIN_SECRET_FILE)
 +    GenerateHmacKey(constants.CLUSTER_DOMAIN_SECRET_FILE)
  
  
  def _InitGanetiServerSetup(master_name):

diff --cc lib/cmdlib.py
index 55749cd,783fedf..1e45225
--- 1/lib/cmdlib.py
--- 2/lib/cmdlib.py
+++ b/lib/cmdlib.py
@@@ -6640,16 -6647,20 +6658,19 @@@ class LUCreateInstance(LogicalUnit)
      else:
        network_port = None
  
-     # this is needed because os.path.join does not accept None arguments
-     if self.op.file_storage_dir is None:
-       string_file_storage_dir = ""
-     else:
-       string_file_storage_dir = self.op.file_storage_dir
+     if constants.ENABLE_FILE_STORAGE:
+       # this is needed because os.path.join does not accept None arguments
+       if self.op.file_storage_dir is None:
+         string_file_storage_dir = ""
+       else:
+         string_file_storage_dir = self.op.file_storage_dir
  
-     # build the full file storage dir path
-     file_storage_dir = utils.PathJoin(self.cfg.GetFileStorageDir(),
-                                       string_file_storage_dir, instance)
+       # build the full file storage dir path
+       file_storage_dir = utils.PathJoin(self.cfg.GetFileStorageDir(),
+                                         string_file_storage_dir, instance)
+     else:
+       file_storage_dir = ""
  
 -
      disks = _GenerateDiskTemplate(self,
                                    self.op.disk_template,
                                    instance, pnode_name,

diff --cc lib/rapi/rlib2.py
Simple merge

diff --cc lib/utils.py
index 97d2c96,d0d07bc..9076558
--- 1/lib/utils.py
--- 2/lib/utils.py
+++ b/lib/utils.py
@@@ -44,10 -43,9 +44,11 @@@ import resourc
  import logging
  import logging.handlers
  import signal
 +import OpenSSL
  import datetime
  import calendar
 +import hmac
+ import collections
  
  from cStringIO import StringIO
  

diff --cc qa/ganeti-qa.py
Simple merge

diff --cc qa/qa_cluster.py
Simple merge

diff --cc test/ganeti.utils_unittest.py
Simple merge

diff --cc tools/cfgupgrade
index c9db1ce,2143ba0..9fd8c68
--- 1/tools/cfgupgrade
--- 2/tools/cfgupgrade
+++ b/tools/cfgupgrade
@@@ -174,7 -174,10 +174,10 @@@ def main()
                      backup=True)
  
      if not options.dry_run:
-       bootstrap.GenerateClusterCrypto(False, False, False, False)
 -      bootstrap.GenerateClusterCrypto(False, False, False,
++      bootstrap.GenerateClusterCrypto(False, False, False, False,
+                                       nodecert_file=options.SERVER_PEM_PATH,
+                                       rapicert_file=options.RAPI_CERT_FILE,
+                                       hmackey_file=options.CONFD_HMAC_KEY)
  
    except:
      logging.critical("Writing configuration failed. It is probably in an"