Statistics
| Branch: | Tag: | Revision:

root / lib / ovf.py @ fa337742

History | View | Annotate | Download (62.5 kB)

1
#!/usr/bin/python
2
#
3

    
4
# Copyright (C) 2011 Google Inc.
5
#
6
# This program is free software; you can redistribute it and/or modify
7
# it under the terms of the GNU General Public License as published by
8
# the Free Software Foundation; either version 2 of the License, or
9
# (at your option) any later version.
10
#
11
# This program is distributed in the hope that it will be useful, but
12
# WITHOUT ANY WARRANTY; without even the implied warranty of
13
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
14
# General Public License for more details.
15
#
16
# You should have received a copy of the GNU General Public License
17
# along with this program; if not, write to the Free Software
18
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
19
# 02110-1301, USA.
20

    
21

    
22
"""Converter tools between ovf and ganeti config file
23

24
"""
25

    
26
# pylint: disable=F0401, E1101
27

    
28
# F0401 because ElementTree is not default for python 2.4
29
# E1101 makes no sense - pylint assumes that ElementTree object is a tuple
30

    
31

    
32
import ConfigParser
33
import errno
34
import logging
35
import os
36
import os.path
37
import re
38
import shutil
39
import tarfile
40
import tempfile
41
import xml.dom.minidom
42
import xml.parsers.expat
43
try:
44
  import xml.etree.ElementTree as ET
45
except ImportError:
46
  import elementtree.ElementTree as ET
47

    
48
from ganeti import constants
49
from ganeti import errors
50
from ganeti import utils
51

    
52

    
53
# Schemas used in OVF format
54
GANETI_SCHEMA = "http://ganeti"
55
OVF_SCHEMA = "http://schemas.dmtf.org/ovf/envelope/1"
56
RASD_SCHEMA = ("http://schemas.dmtf.org/wbem/wscim/1/cim-schema/2/"
57
               "CIM_ResourceAllocationSettingData")
58
VSSD_SCHEMA = ("http://schemas.dmtf.org/wbem/wscim/1/cim-schema/2/"
59
               "CIM_VirtualSystemSettingData")
60
XML_SCHEMA = "http://www.w3.org/2001/XMLSchema-instance"
61

    
62
# File extensions in OVF package
63
OVA_EXT = ".ova"
64
OVF_EXT = ".ovf"
65
MF_EXT = ".mf"
66
CERT_EXT = ".cert"
67
COMPRESSION_EXT = ".gz"
68
FILE_EXTENSIONS = [
69
  OVF_EXT,
70
  MF_EXT,
71
  CERT_EXT,
72
]
73

    
74
COMPRESSION_TYPE = "gzip"
75
NO_COMPRESSION = [None, "identity"]
76
COMPRESS = "compression"
77
DECOMPRESS = "decompression"
78
ALLOWED_ACTIONS = [COMPRESS, DECOMPRESS]
79

    
80
VMDK = "vmdk"
81
RAW = "raw"
82
COW = "cow"
83
ALLOWED_FORMATS = [RAW, COW, VMDK]
84

    
85
# ResourceType values
86
RASD_TYPE = {
87
  "vcpus": "3",
88
  "memory": "4",
89
  "scsi-controller": "6",
90
  "ethernet-adapter": "10",
91
  "disk": "17",
92
}
93

    
94
SCSI_SUBTYPE = "lsilogic"
95
VS_TYPE = {
96
  "ganeti": "ganeti-ovf",
97
  "external": "vmx-04",
98
}
99

    
100
# AllocationUnits values and conversion
101
ALLOCATION_UNITS = {
102
  'b': ["bytes", "b"],
103
  'kb': ["kilobytes", "kb", "byte * 2^10", "kibibytes", "kib"],
104
  'mb': ["megabytes", "mb", "byte * 2^20", "mebibytes", "mib"],
105
  'gb': ["gigabytes", "gb", "byte * 2^30", "gibibytes", "gib"],
106
}
107
CONVERT_UNITS_TO_MB = {
108
  'b': lambda x: x / (1024 * 1024),
109
  'kb': lambda x: x / 1024,
110
  'mb': lambda x: x,
111
  'gb': lambda x: x * 1024,
112
}
113

    
114
# Names of the config fields
115
NAME = "name"
116
OS = "os"
117
HYPERV = "hypervisor"
118
VCPUS = "vcpus"
119
MEMORY = "memory"
120
AUTO_BALANCE = "auto_balance"
121
DISK_TEMPLATE = "disk_template"
122
TAGS = "tags"
123
VERSION = "version"
124

    
125
# Instance IDs of System and SCSI controller
126
INSTANCE_ID = {
127
  "system": 0,
128
  "vcpus": 1,
129
  "memory": 2,
130
  "scsi": 3,
131
}
132

    
133
# Disk format descriptions
134
DISK_FORMAT = {
135
  RAW: "http://en.wikipedia.org/wiki/Byte",
136
  VMDK: "http://www.vmware.com/interfaces/specifications/vmdk.html"
137
          "#monolithicSparse",
138
  COW: "http://www.gnome.org/~markmc/qcow-image-format.html",
139
}
140

    
141

    
142
def LinkFile(old_path, prefix=None, suffix=None, directory=None):
143
  """Create link with a given prefix and suffix.
144

145
  This is a wrapper over os.link. It tries to create a hard link for given file,
146
  but instead of rising error when file exists, the function changes the name
147
  a little bit.
148

149
  @type old_path:string
150
  @param old_path: path to the file that is to be linked
151
  @type prefix: string
152
  @param prefix: prefix of filename for the link
153
  @type suffix: string
154
  @param suffix: suffix of the filename for the link
155
  @type directory: string
156
  @param directory: directory of the link
157

158
  @raise errors.OpPrereqError: when error on linking is different than
159
    "File exists"
160

161
  """
162
  assert(prefix is not None or suffix is not None)
163
  if directory is None:
164
    directory = os.getcwd()
165
  new_path = utils.PathJoin(directory, "%s%s" % (prefix, suffix))
166
  counter = 1
167
  while True:
168
    try:
169
      os.link(old_path, new_path)
170
      break
171
    except OSError, err:
172
      if err.errno == errno.EEXIST:
173
        new_path = utils.PathJoin(directory,
174
          "%s_%s%s" % (prefix, counter, suffix))
175
        counter += 1
176
      else:
177
        raise errors.OpPrereqError("Error moving the file %s to %s location:"
178
                                   " %s" % (old_path, new_path, err))
179
  return new_path
180

    
181

    
182
class OVFReader(object):
183
  """Reader class for OVF files.
184

185
  @type files_list: list
186
  @ivar files_list: list of files in the OVF package
187
  @type tree: ET.ElementTree
188
  @ivar tree: XML tree of the .ovf file
189
  @type schema_name: string
190
  @ivar schema_name: name of the .ovf file
191
  @type input_dir: string
192
  @ivar input_dir: directory in which the .ovf file resides
193

194
  """
195
  def __init__(self, input_path):
196
    """Initialiaze the reader - load the .ovf file to XML parser.
197

198
    It is assumed that names of manifesto (.mf), certificate (.cert) and ovf
199
    files are the same. In order to account any other files as part of the ovf
200
    package, they have to be explicitly mentioned in the Resources section
201
    of the .ovf file.
202

203
    @type input_path: string
204
    @param input_path: absolute path to the .ovf file
205

206
    @raise errors.OpPrereqError: when .ovf file is not a proper XML file or some
207
      of the files mentioned in Resources section do not exist
208

209
    """
210
    self.tree = ET.ElementTree()
211
    try:
212
      self.tree.parse(input_path)
213
    except xml.parsers.expat.ExpatError, err:
214
      raise errors.OpPrereqError("Error while reading %s file: %s" %
215
                                 (OVF_EXT, err))
216

    
217
    # Create a list of all files in the OVF package
218
    (input_dir, input_file) = os.path.split(input_path)
219
    (input_name, _) = os.path.splitext(input_file)
220
    files_directory = utils.ListVisibleFiles(input_dir)
221
    files_list = []
222
    for file_name in files_directory:
223
      (name, extension) = os.path.splitext(file_name)
224
      if extension in FILE_EXTENSIONS and name == input_name:
225
        files_list.append(file_name)
226
    files_list += self._GetAttributes("{%s}References/{%s}File" %
227
                                      (OVF_SCHEMA, OVF_SCHEMA),
228
                                      "{%s}href" % OVF_SCHEMA)
229
    for file_name in files_list:
230
      file_path = utils.PathJoin(input_dir, file_name)
231
      if not os.path.exists(file_path):
232
        raise errors.OpPrereqError("File does not exist: %s" % file_path)
233
    logging.info("Files in the OVF package: %s", " ".join(files_list))
234
    self.files_list = files_list
235
    self.input_dir = input_dir
236
    self.schema_name = input_name
237

    
238
  def _GetAttributes(self, path, attribute):
239
    """Get specified attribute from all nodes accessible using given path.
240

241
    Function follows the path from root node to the desired tags using path,
242
    then reads the apropriate attribute values.
243

244
    @type path: string
245
    @param path: path of nodes to visit
246
    @type attribute: string
247
    @param attribute: attribute for which we gather the information
248
    @rtype: list
249
    @return: for each accessible tag with the attribute value set, value of the
250
      attribute
251

252
    """
253
    current_list = self.tree.findall(path)
254
    results = [x.get(attribute) for x in current_list]
255
    return filter(None, results)
256

    
257
  def _GetElementMatchingAttr(self, path, match_attr):
258
    """Searches for element on a path that matches certain attribute value.
259

260
    Function follows the path from root node to the desired tags using path,
261
    then searches for the first one matching the attribute value.
262

263
    @type path: string
264
    @param path: path of nodes to visit
265
    @type match_attr: tuple
266
    @param match_attr: pair (attribute, value) for which we search
267
    @rtype: ET.ElementTree or None
268
    @return: first element matching match_attr or None if nothing matches
269

270
    """
271
    potential_elements = self.tree.findall(path)
272
    (attr, val) = match_attr
273
    for elem in potential_elements:
274
      if elem.get(attr) == val:
275
        return elem
276
    return None
277

    
278
  def _GetElementMatchingText(self, path, match_text):
279
    """Searches for element on a path that matches certain text value.
280

281
    Function follows the path from root node to the desired tags using path,
282
    then searches for the first one matching the text value.
283

284
    @type path: string
285
    @param path: path of nodes to visit
286
    @type match_text: tuple
287
    @param match_text: pair (node, text) for which we search
288
    @rtype: ET.ElementTree or None
289
    @return: first element matching match_text or None if nothing matches
290

291
    """
292
    potential_elements = self.tree.findall(path)
293
    (node, text) = match_text
294
    for elem in potential_elements:
295
      if elem.findtext(node) == text:
296
        return elem
297
    return None
298

    
299
  @staticmethod
300
  def _GetDictParameters(root, schema):
301
    """Reads text in all children and creates the dictionary from the contents.
302

303
    @type root: ET.ElementTree or None
304
    @param root: father of the nodes we want to collect data about
305
    @type schema: string
306
    @param schema: schema name to be removed from the tag
307
    @rtype: dict
308
    @return: dictionary containing tags and their text contents, tags have their
309
      schema fragment removed or empty dictionary, when root is None
310

311
    """
312
    if not root:
313
      return {}
314
    results = {}
315
    for element in list(root):
316
      pref_len = len("{%s}" % schema)
317
      assert(schema in element.tag)
318
      tag = element.tag[pref_len:]
319
      results[tag] = element.text
320
    return results
321

    
322
  def VerifyManifest(self):
323
    """Verifies manifest for the OVF package, if one is given.
324

325
    @raise errors.OpPrereqError: if SHA1 checksums do not match
326

327
    """
328
    if "%s%s" % (self.schema_name, MF_EXT) in self.files_list:
329
      logging.warning("Verifying SHA1 checksums, this may take a while")
330
      manifest_filename = "%s%s" % (self.schema_name, MF_EXT)
331
      manifest_path = utils.PathJoin(self.input_dir, manifest_filename)
332
      manifest_content = utils.ReadFile(manifest_path).splitlines()
333
      manifest_files = {}
334
      regexp = r"SHA1\((\S+)\)= (\S+)"
335
      for line in manifest_content:
336
        match = re.match(regexp, line)
337
        if match:
338
          file_name = match.group(1)
339
          sha1_sum = match.group(2)
340
          manifest_files[file_name] = sha1_sum
341
      files_with_paths = [utils.PathJoin(self.input_dir, file_name)
342
        for file_name in self.files_list]
343
      sha1_sums = utils.FingerprintFiles(files_with_paths)
344
      for file_name, value in manifest_files.iteritems():
345
        if sha1_sums.get(utils.PathJoin(self.input_dir, file_name)) != value:
346
          raise errors.OpPrereqError("SHA1 checksum of %s does not match the"
347
                                     " value in manifest file" % file_name)
348
      logging.info("SHA1 checksums verified")
349

    
350
  def GetInstanceName(self):
351
    """Provides information about instance name.
352

353
    @rtype: string
354
    @return: instance name string
355

356
    """
357
    find_name = "{%s}VirtualSystem/{%s}Name" % (OVF_SCHEMA, OVF_SCHEMA)
358
    return self.tree.findtext(find_name)
359

    
360
  def GetDiskTemplate(self):
361
    """Returns disk template from .ovf file
362

363
    @rtype: string or None
364
    @return: name of the template
365
    """
366
    find_template = ("{%s}GanetiSection/{%s}DiskTemplate" %
367
                     (GANETI_SCHEMA, GANETI_SCHEMA))
368
    return self.tree.findtext(find_template)
369

    
370
  def GetHypervisorData(self):
371
    """Provides hypervisor information - hypervisor name and options.
372

373
    @rtype: dict
374
    @return: dictionary containing name of the used hypervisor and all the
375
      specified options
376

377
    """
378
    hypervisor_search = ("{%s}GanetiSection/{%s}Hypervisor" %
379
                         (GANETI_SCHEMA, GANETI_SCHEMA))
380
    hypervisor_data = self.tree.find(hypervisor_search)
381
    if not hypervisor_data:
382
      return {"hypervisor_name": constants.VALUE_AUTO}
383
    results = {
384
      "hypervisor_name": hypervisor_data.findtext("{%s}Name" % GANETI_SCHEMA,
385
                           default=constants.VALUE_AUTO),
386
    }
387
    parameters = hypervisor_data.find("{%s}Parameters" % GANETI_SCHEMA)
388
    results.update(self._GetDictParameters(parameters, GANETI_SCHEMA))
389
    return results
390

    
391
  def GetOSData(self):
392
    """ Provides operating system information - os name and options.
393

394
    @rtype: dict
395
    @return: dictionary containing name and options for the chosen OS
396

397
    """
398
    results = {}
399
    os_search = ("{%s}GanetiSection/{%s}OperatingSystem" %
400
                 (GANETI_SCHEMA, GANETI_SCHEMA))
401
    os_data = self.tree.find(os_search)
402
    if os_data:
403
      results["os_name"] = os_data.findtext("{%s}Name" % GANETI_SCHEMA)
404
      parameters = os_data.find("{%s}Parameters" % GANETI_SCHEMA)
405
      results.update(self._GetDictParameters(parameters, GANETI_SCHEMA))
406
    return results
407

    
408
  def GetBackendData(self):
409
    """ Provides backend information - vcpus, memory, auto balancing options.
410

411
    @rtype: dict
412
    @return: dictionary containing options for vcpus, memory and auto balance
413
      settings
414

415
    """
416
    results = {}
417

    
418
    find_vcpus = ("{%s}VirtualSystem/{%s}VirtualHardwareSection/{%s}Item" %
419
                   (OVF_SCHEMA, OVF_SCHEMA, OVF_SCHEMA))
420
    match_vcpus = ("{%s}ResourceType" % RASD_SCHEMA, RASD_TYPE["vcpus"])
421
    vcpus = self._GetElementMatchingText(find_vcpus, match_vcpus)
422
    if vcpus:
423
      vcpus_count = vcpus.findtext("{%s}VirtualQuantity" % RASD_SCHEMA,
424
        default=constants.VALUE_AUTO)
425
    else:
426
      vcpus_count = constants.VALUE_AUTO
427
    results["vcpus"] = str(vcpus_count)
428

    
429
    find_memory = find_vcpus
430
    match_memory = ("{%s}ResourceType" % RASD_SCHEMA, RASD_TYPE["memory"])
431
    memory = self._GetElementMatchingText(find_memory, match_memory)
432
    memory_raw = None
433
    if memory:
434
      alloc_units = memory.findtext("{%s}AllocationUnits" % RASD_SCHEMA)
435
      matching_units = [units for units, variants in
436
        ALLOCATION_UNITS.iteritems() if alloc_units.lower() in variants]
437
      if matching_units == []:
438
        raise errors.OpPrereqError("Unit %s for RAM memory unknown",
439
          alloc_units)
440
      units = matching_units[0]
441
      memory_raw = int(memory.findtext("{%s}VirtualQuantity" % RASD_SCHEMA,
442
            default=constants.VALUE_AUTO))
443
      memory_count = CONVERT_UNITS_TO_MB[units](memory_raw)
444
    else:
445
      memory_count = constants.VALUE_AUTO
446
    results["memory"] = str(memory_count)
447

    
448
    find_balance = ("{%s}GanetiSection/{%s}AutoBalance" %
449
                   (GANETI_SCHEMA, GANETI_SCHEMA))
450
    balance = self.tree.findtext(find_balance, default=constants.VALUE_AUTO)
451
    results["auto_balance"] = balance
452

    
453
    return results
454

    
455
  def GetTagsData(self):
456
    """Provides tags information for instance.
457

458
    @rtype: string or None
459
    @return: string of comma-separated tags for the instance
460

461
    """
462
    find_tags = "{%s}GanetiSection/{%s}Tags" % (GANETI_SCHEMA, GANETI_SCHEMA)
463
    results = self.tree.findtext(find_tags)
464
    if results:
465
      return results
466
    else:
467
      return None
468

    
469
  def GetVersionData(self):
470
    """Provides version number read from .ovf file
471

472
    @rtype: string
473
    @return: string containing the version number
474

475
    """
476
    find_version = ("{%s}GanetiSection/{%s}Version" %
477
                    (GANETI_SCHEMA, GANETI_SCHEMA))
478
    return self.tree.findtext(find_version)
479

    
480
  def GetNetworkData(self):
481
    """Provides data about the network in the OVF instance.
482

483
    The method gathers the data about networks used by OVF instance. It assumes
484
    that 'name' tag means something - in essence, if it contains one of the
485
    words 'bridged' or 'routed' then that will be the mode of this network in
486
    Ganeti. The information about the network can be either in GanetiSection or
487
    VirtualHardwareSection.
488

489
    @rtype: dict
490
    @return: dictionary containing all the network information
491

492
    """
493
    results = {}
494
    networks_search = ("{%s}NetworkSection/{%s}Network" %
495
                       (OVF_SCHEMA, OVF_SCHEMA))
496
    network_names = self._GetAttributes(networks_search,
497
      "{%s}name" % OVF_SCHEMA)
498
    required = ["ip", "mac", "link", "mode"]
499
    for (counter, network_name) in enumerate(network_names):
500
      network_search = ("{%s}VirtualSystem/{%s}VirtualHardwareSection/{%s}Item"
501
                        % (OVF_SCHEMA, OVF_SCHEMA, OVF_SCHEMA))
502
      ganeti_search = ("{%s}GanetiSection/{%s}Network/{%s}Nic" %
503
                       (GANETI_SCHEMA, GANETI_SCHEMA, GANETI_SCHEMA))
504
      network_match = ("{%s}Connection" % RASD_SCHEMA, network_name)
505
      ganeti_match = ("{%s}name" % OVF_SCHEMA, network_name)
506
      network_data = self._GetElementMatchingText(network_search, network_match)
507
      network_ganeti_data = self._GetElementMatchingAttr(ganeti_search,
508
        ganeti_match)
509

    
510
      ganeti_data = {}
511
      if network_ganeti_data:
512
        ganeti_data["mode"] = network_ganeti_data.findtext("{%s}Mode" %
513
                                                           GANETI_SCHEMA)
514
        ganeti_data["mac"] = network_ganeti_data.findtext("{%s}MACAddress" %
515
                                                          GANETI_SCHEMA)
516
        ganeti_data["ip"] = network_ganeti_data.findtext("{%s}IPAddress" %
517
                                                         GANETI_SCHEMA)
518
        ganeti_data["link"] = network_ganeti_data.findtext("{%s}Link" %
519
                                                           GANETI_SCHEMA)
520
      mac_data = None
521
      if network_data:
522
        mac_data = network_data.findtext("{%s}Address" % RASD_SCHEMA)
523

    
524
      network_name = network_name.lower()
525

    
526
      # First, some not Ganeti-specific information is collected
527
      if constants.NIC_MODE_BRIDGED in network_name:
528
        results["nic%s_mode" % counter] = "bridged"
529
      elif constants.NIC_MODE_ROUTED in network_name:
530
        results["nic%s_mode" % counter] = "routed"
531
      results["nic%s_mac" % counter] = mac_data
532

    
533
      # GanetiSection data overrides 'manually' collected data
534
      for name, value in ganeti_data.iteritems():
535
        results["nic%s_%s" % (counter, name)] = value
536

    
537
      # Bridged network has no IP - unless specifically stated otherwise
538
      if (results.get("nic%s_mode" % counter) == "bridged" and
539
          not results.get("nic%s_ip" % counter)):
540
        results["nic%s_ip" % counter] = constants.VALUE_NONE
541

    
542
      for option in required:
543
        if not results.get("nic%s_%s" % (counter, option)):
544
          results["nic%s_%s" % (counter, option)] = constants.VALUE_AUTO
545

    
546
    if network_names:
547
      results["nic_count"] = str(len(network_names))
548
    return results
549

    
550
  def GetDisksNames(self):
551
    """Provides list of file names for the disks used by the instance.
552

553
    @rtype: list
554
    @return: list of file names, as referenced in .ovf file
555

556
    """
557
    results = []
558
    disks_search = "{%s}DiskSection/{%s}Disk" % (OVF_SCHEMA, OVF_SCHEMA)
559
    disk_ids = self._GetAttributes(disks_search, "{%s}fileRef" % OVF_SCHEMA)
560
    for disk in disk_ids:
561
      disk_search = "{%s}References/{%s}File" % (OVF_SCHEMA, OVF_SCHEMA)
562
      disk_match = ("{%s}id" % OVF_SCHEMA, disk)
563
      disk_elem = self._GetElementMatchingAttr(disk_search, disk_match)
564
      if disk_elem is None:
565
        raise errors.OpPrereqError("%s file corrupted - disk %s not found in"
566
                                   " references" % (OVF_EXT, disk))
567
      disk_name = disk_elem.get("{%s}href" % OVF_SCHEMA)
568
      disk_compression = disk_elem.get("{%s}compression" % OVF_SCHEMA)
569
      results.append((disk_name, disk_compression))
570
    return results
571

    
572

    
573
def SubElementText(parent, tag, text, attrib={}, **extra):
574
# pylint: disable=W0102
575
  """This is just a wrapper on ET.SubElement that always has text content.
576

577
  """
578
  if text is None:
579
    return None
580
  elem = ET.SubElement(parent, tag, attrib=attrib, **extra)
581
  elem.text = str(text)
582
  return elem
583

    
584

    
585
class OVFWriter(object):
586
  """Writer class for OVF files.
587

588
  @type tree: ET.ElementTree
589
  @ivar tree: XML tree that we are constructing
590
  @type virtual_system_type: string
591
  @ivar virtual_system_type: value of vssd:VirtualSystemType, for external usage
592
    in VMWare this requires to be vmx
593
  @type hardware_list: list
594
  @ivar hardware_list: list of items prepared for VirtualHardwareSection
595
  @type next_instance_id: int
596
  @ivar next_instance_id: next instance id to be used when creating elements on
597
    hardware_list
598

599
  """
600
  def __init__(self, has_gnt_section):
601
    """Initialize the writer - set the top element.
602

603
    @type has_gnt_section: bool
604
    @param has_gnt_section: if the Ganeti schema should be added - i.e. this
605
      means that Ganeti section will be present
606

607
    """
608
    env_attribs = {
609
      "xmlns:xsi": XML_SCHEMA,
610
      "xmlns:vssd": VSSD_SCHEMA,
611
      "xmlns:rasd": RASD_SCHEMA,
612
      "xmlns:ovf": OVF_SCHEMA,
613
      "xmlns": OVF_SCHEMA,
614
      "xml:lang": "en-US",
615
    }
616
    if has_gnt_section:
617
      env_attribs["xmlns:gnt"] = GANETI_SCHEMA
618
      self.virtual_system_type = VS_TYPE["ganeti"]
619
    else:
620
      self.virtual_system_type = VS_TYPE["external"]
621
    self.tree = ET.Element("Envelope", attrib=env_attribs)
622
    self.hardware_list = []
623
    # INSTANCE_ID contains statically assigned IDs, starting from 0
624
    self.next_instance_id = len(INSTANCE_ID) # FIXME: hackish
625

    
626
  def SaveDisksData(self, disks):
627
    """Convert disk information to certain OVF sections.
628

629
    @type disks: list
630
    @param disks: list of dictionaries of disk options from config.ini
631

632
    """
633
    references = ET.SubElement(self.tree, "References")
634
    disk_section = ET.SubElement(self.tree, "DiskSection")
635
    SubElementText(disk_section, "Info", "Virtual disk information")
636
    for counter, disk in enumerate(disks):
637
      file_id = "file%s" % counter
638
      disk_id = "disk%s" % counter
639
      file_attribs = {
640
        "ovf:href": disk["path"],
641
        "ovf:size": str(disk["real-size"]),
642
        "ovf:id": file_id,
643
      }
644
      disk_attribs = {
645
        "ovf:capacity": str(disk["virt-size"]),
646
        "ovf:diskId": disk_id,
647
        "ovf:fileRef": file_id,
648
        "ovf:format": DISK_FORMAT.get(disk["format"], disk["format"]),
649
      }
650
      if "compression" in disk:
651
        file_attribs["ovf:compression"] = disk["compression"]
652
      ET.SubElement(references, "File", attrib=file_attribs)
653
      ET.SubElement(disk_section, "Disk", attrib=disk_attribs)
654

    
655
      # Item in VirtualHardwareSection creation
656
      disk_item = ET.Element("Item")
657
      SubElementText(disk_item, "rasd:ElementName", disk_id)
658
      SubElementText(disk_item, "rasd:HostResource", "ovf:/disk/%s" % disk_id)
659
      SubElementText(disk_item, "rasd:InstanceID", self.next_instance_id)
660
      SubElementText(disk_item, "rasd:Parent", INSTANCE_ID["scsi"])
661
      SubElementText(disk_item, "rasd:ResourceType", RASD_TYPE["disk"])
662
      self.hardware_list.append(disk_item)
663
      self.next_instance_id += 1
664

    
665
  def SaveNetworksData(self, networks):
666
    """Convert network information to NetworkSection.
667

668
    @type networks: list
669
    @param networks: list of dictionaries of network options form config.ini
670

671
    """
672
    network_section = ET.SubElement(self.tree, "NetworkSection")
673
    SubElementText(network_section, "Info", "List of logical networks")
674
    for counter, network in enumerate(networks):
675
      network_name = "%s%s" % (network["mode"], counter)
676
      network_attrib = {"ovf:name": network_name}
677
      ET.SubElement(network_section, "Network", attrib=network_attrib)
678

    
679
      # Item in VirtualHardwareSection creation
680
      network_item = ET.Element("Item")
681
      SubElementText(network_item, "rasd:Address", network["mac"])
682
      SubElementText(network_item, "rasd:Connection", network_name)
683
      SubElementText(network_item, "rasd:ElementName", network_name)
684
      SubElementText(network_item, "rasd:InstanceID", self.next_instance_id)
685
      SubElementText(network_item, "rasd:ResourceType",
686
        RASD_TYPE["ethernet-adapter"])
687
      self.hardware_list.append(network_item)
688
      self.next_instance_id += 1
689

    
690
  @staticmethod
691
  def _SaveNameAndParams(root, data):
692
    """Save name and parameters information under root using data.
693

694
    @type root: ET.Element
695
    @param root: root element for the Name and Parameters
696
    @type data: dict
697
    @param data: data from which we gather the values
698

699
    """
700
    assert(data.get("name"))
701
    name = SubElementText(root, "gnt:Name", data["name"])
702
    params = ET.SubElement(root, "gnt:Parameters")
703
    for name, value in data.iteritems():
704
      if name != "name":
705
        SubElementText(params, "gnt:%s" % name, value)
706

    
707
  def SaveGanetiData(self, ganeti, networks):
708
    """Convert Ganeti-specific information to GanetiSection.
709

710
    @type ganeti: dict
711
    @param ganeti: dictionary of Ganeti-specific options from config.ini
712
    @type networks: list
713
    @param networks: list of dictionaries of network options form config.ini
714

715
    """
716
    ganeti_section = ET.SubElement(self.tree, "gnt:GanetiSection")
717

    
718
    SubElementText(ganeti_section, "gnt:Version", ganeti.get("version"))
719
    SubElementText(ganeti_section, "gnt:DiskTemplate",
720
      ganeti.get("disk_template"))
721
    SubElementText(ganeti_section, "gnt:AutoBalance",
722
      ganeti.get("auto_balance"))
723
    SubElementText(ganeti_section, "gnt:Tags", ganeti.get("tags"))
724

    
725
    osys = ET.SubElement(ganeti_section, "gnt:OperatingSystem")
726
    self._SaveNameAndParams(osys, ganeti["os"])
727

    
728
    hypervisor = ET.SubElement(ganeti_section, "gnt:Hypervisor")
729
    self._SaveNameAndParams(hypervisor, ganeti["hypervisor"])
730

    
731
    network_section = ET.SubElement(ganeti_section, "gnt:Network")
732
    for counter, network in enumerate(networks):
733
      network_name = "%s%s" % (network["mode"], counter)
734
      nic_attrib = {"ovf:name": network_name}
735
      nic = ET.SubElement(network_section, "gnt:Nic", attrib=nic_attrib)
736
      SubElementText(nic, "gnt:Mode", network["mode"])
737
      SubElementText(nic, "gnt:MACAddress", network["mac"])
738
      SubElementText(nic, "gnt:IPAddress", network["ip"])
739
      SubElementText(nic, "gnt:Link", network["link"])
740

    
741
  def SaveVirtualSystemData(self, name, vcpus, memory):
742
    """Convert virtual system information to OVF sections.
743

744
    @type name: string
745
    @param name: name of the instance
746
    @type vcpus: int
747
    @param vcpus: number of VCPUs
748
    @type memory: int
749
    @param memory: RAM memory in MB
750

751
    """
752
    assert(vcpus > 0)
753
    assert(memory > 0)
754
    vs_attrib = {"ovf:id": name}
755
    virtual_system = ET.SubElement(self.tree, "VirtualSystem", attrib=vs_attrib)
756
    SubElementText(virtual_system, "Info", "A virtual machine")
757

    
758
    name_section = ET.SubElement(virtual_system, "Name")
759
    name_section.text = name
760
    os_attrib = {"ovf:id": "0"}
761
    os_section = ET.SubElement(virtual_system, "OperatingSystemSection",
762
      attrib=os_attrib)
763
    SubElementText(os_section, "Info", "Installed guest operating system")
764
    hardware_section = ET.SubElement(virtual_system, "VirtualHardwareSection")
765
    SubElementText(hardware_section, "Info", "Virtual hardware requirements")
766

    
767
    # System description
768
    system = ET.SubElement(hardware_section, "System")
769
    SubElementText(system, "vssd:ElementName", "Virtual Hardware Family")
770
    SubElementText(system, "vssd:InstanceID", INSTANCE_ID["system"])
771
    SubElementText(system, "vssd:VirtualSystemIdentifier", name)
772
    SubElementText(system, "vssd:VirtualSystemType", self.virtual_system_type)
773

    
774
    # Item for vcpus
775
    vcpus_item = ET.SubElement(hardware_section, "Item")
776
    SubElementText(vcpus_item, "rasd:ElementName",
777
      "%s virtual CPU(s)" % vcpus)
778
    SubElementText(vcpus_item, "rasd:InstanceID", INSTANCE_ID["vcpus"])
779
    SubElementText(vcpus_item, "rasd:ResourceType", RASD_TYPE["vcpus"])
780
    SubElementText(vcpus_item, "rasd:VirtualQuantity", vcpus)
781

    
782
    # Item for memory
783
    memory_item = ET.SubElement(hardware_section, "Item")
784
    SubElementText(memory_item, "rasd:AllocationUnits", "byte * 2^20")
785
    SubElementText(memory_item, "rasd:ElementName", "%sMB of memory" % memory)
786
    SubElementText(memory_item, "rasd:InstanceID", INSTANCE_ID["memory"])
787
    SubElementText(memory_item, "rasd:ResourceType", RASD_TYPE["memory"])
788
    SubElementText(memory_item, "rasd:VirtualQuantity", memory)
789

    
790
    # Item for scsi controller
791
    scsi_item = ET.SubElement(hardware_section, "Item")
792
    SubElementText(scsi_item, "rasd:Address", INSTANCE_ID["system"])
793
    SubElementText(scsi_item, "rasd:ElementName", "scsi_controller0")
794
    SubElementText(scsi_item, "rasd:InstanceID", INSTANCE_ID["scsi"])
795
    SubElementText(scsi_item, "rasd:ResourceSubType", SCSI_SUBTYPE)
796
    SubElementText(scsi_item, "rasd:ResourceType", RASD_TYPE["scsi-controller"])
797

    
798
    # Other items - from self.hardware_list
799
    for item in self.hardware_list:
800
      hardware_section.append(item)
801

    
802
  def PrettyXmlDump(self):
803
    """Formatter of the XML file.
804

805
    @rtype: string
806
    @return: XML tree in the form of nicely-formatted string
807

808
    """
809
    raw_string = ET.tostring(self.tree)
810
    parsed_xml = xml.dom.minidom.parseString(raw_string)
811
    xml_string = parsed_xml.toprettyxml(indent="  ")
812
    text_re = re.compile(">\n\s+([^<>\s].*?)\n\s+</", re.DOTALL)
813
    return text_re.sub(">\g<1></", xml_string)
814

    
815

    
816
class Converter(object):
817
  """Converter class for OVF packages.
818

819
  Converter is a class above both ImporterOVF and ExporterOVF. It's purpose is
820
  to provide a common interface for the two.
821

822
  @type options: optparse.Values
823
  @ivar options: options parsed from the command line
824
  @type output_dir: string
825
  @ivar output_dir: directory to which the results of conversion shall be
826
    written
827
  @type temp_file_manager: L{utils.TemporaryFileManager}
828
  @ivar temp_file_manager: container for temporary files created during
829
    conversion
830
  @type temp_dir: string
831
  @ivar temp_dir: temporary directory created then we deal with OVA
832

833
  """
834
  def __init__(self, input_path, options):
835
    """Initialize the converter.
836

837
    @type input_path: string
838
    @param input_path: path to the Converter input file
839
    @type options: optparse.Values
840
    @param options: command line options
841

842
    @raise errors.OpPrereqError: if file does not exist
843

844
    """
845
    input_path = os.path.abspath(input_path)
846
    if not os.path.isfile(input_path):
847
      raise errors.OpPrereqError("File does not exist: %s" % input_path)
848
    self.options = options
849
    self.temp_file_manager = utils.TemporaryFileManager()
850
    self.temp_dir = None
851
    self.output_dir = None
852
    self._ReadInputData(input_path)
853

    
854
  def _ReadInputData(self, input_path):
855
    """Reads the data on which the conversion will take place.
856

857
    @type input_path: string
858
    @param input_path: absolute path to the Converter input file
859

860
    """
861
    raise NotImplementedError()
862

    
863
  def _CompressDisk(self, disk_path, compression, action):
864
    """Performs (de)compression on the disk and returns the new path
865

866
    @type disk_path: string
867
    @param disk_path: path to the disk
868
    @type compression: string
869
    @param compression: compression type
870
    @type action: string
871
    @param action: whether the action is compression or decompression
872
    @rtype: string
873
    @return: new disk path after (de)compression
874

875
    @raise errors.OpPrereqError: disk (de)compression failed or "compression"
876
      is not supported
877

878
    """
879
    assert(action in ALLOWED_ACTIONS)
880
    # For now we only support gzip, as it is used in ovftool
881
    if compression != COMPRESSION_TYPE:
882
      raise errors.OpPrereqError("Unsupported compression type: %s"
883
                                 % compression)
884
    disk_file = os.path.basename(disk_path)
885
    if action == DECOMPRESS:
886
      (disk_name, _) = os.path.splitext(disk_file)
887
      prefix = disk_name
888
    elif action == COMPRESS:
889
      prefix = disk_file
890
    new_path = utils.GetClosedTempfile(suffix=COMPRESSION_EXT, prefix=prefix,
891
      dir=self.output_dir)
892
    self.temp_file_manager.Add(new_path)
893
    args = ["gzip", "-c", disk_path]
894
    run_result = utils.RunCmd(args, output=new_path)
895
    if run_result.failed:
896
      raise errors.OpPrereqError("Disk %s failed with output: %s"
897
                                 % (action, run_result.stderr))
898
    logging.info("The %s of the disk is completed", action)
899
    return (COMPRESSION_EXT, new_path)
900

    
901
  def _ConvertDisk(self, disk_format, disk_path):
902
    """Performes conversion to specified format.
903

904
    @type disk_format: string
905
    @param disk_format: format to which the disk should be converted
906
    @type disk_path: string
907
    @param disk_path: path to the disk that should be converted
908
    @rtype: string
909
    @return path to the output disk
910

911
    @raise errors.OpPrereqError: convertion of the disk failed
912

913
    """
914
    disk_file = os.path.basename(disk_path)
915
    (disk_name, disk_extension) = os.path.splitext(disk_file)
916
    if disk_extension != disk_format:
917
      logging.warning("Conversion of disk image to %s format, this may take"
918
                      " a while", disk_format)
919

    
920
    new_disk_path = utils.GetClosedTempfile(suffix=".%s" % disk_format,
921
      prefix=disk_name, dir=self.output_dir)
922
    self.temp_file_manager.Add(new_disk_path)
923
    args = [
924
      "qemu-img",
925
      "convert",
926
      "-O",
927
      disk_format,
928
      disk_path,
929
      new_disk_path,
930
    ]
931
    run_result = utils.RunCmd(args, cwd=os.getcwd())
932
    if run_result.failed:
933
      raise errors.OpPrereqError("Convertion to %s failed, qemu-img output was"
934
                                 ": %s" % (disk_format, run_result.stderr))
935
    return (".%s" % disk_format, new_disk_path)
936

    
937
  @staticmethod
938
  def _GetDiskQemuInfo(disk_path, regexp):
939
    """Figures out some information of the disk using qemu-img.
940

941
    @type disk_path: string
942
    @param disk_path: path to the disk we want to know the format of
943
    @type regexp: string
944
    @param regexp: string that has to be matched, it has to contain one group
945
    @rtype: string
946
    @return: disk format
947

948
    @raise errors.OpPrereqError: format information cannot be retrieved
949

950
    """
951
    args = ["qemu-img", "info", disk_path]
952
    run_result = utils.RunCmd(args, cwd=os.getcwd())
953
    if run_result.failed:
954
      raise errors.OpPrereqError("Gathering info about the disk using qemu-img"
955
                                 " failed, output was: %s" % run_result.stderr)
956
    result = run_result.output
957
    regexp = r"%s" % regexp
958
    match = re.search(regexp, result)
959
    if match:
960
      disk_format = match.group(1)
961
    else:
962
      raise errors.OpPrereqError("No file information matching %s found in:"
963
                                 " %s" % (regexp, result))
964
    return disk_format
965

    
966
  def Parse(self):
967
    """Parses the data and creates a structure containing all required info.
968

969
    """
970
    raise NotImplementedError()
971

    
972
  def Save(self):
973
    """Saves the gathered configuration in an apropriate format.
974

975
    """
976
    raise NotImplementedError()
977

    
978
  def Cleanup(self):
979
    """Cleans the temporary directory, if one was created.
980

981
    """
982
    self.temp_file_manager.Cleanup()
983
    if self.temp_dir:
984
      shutil.rmtree(self.temp_dir)
985
      self.temp_dir = None
986

    
987

    
988
class OVFImporter(Converter):
989
  """Converter from OVF to Ganeti config file.
990

991
  @type input_dir: string
992
  @ivar input_dir: directory in which the .ovf file resides
993
  @type output_dir: string
994
  @ivar output_dir: directory to which the results of conversion shall be
995
    written
996
  @type input_path: string
997
  @ivar input_path: complete path to the .ovf file
998
  @type ovf_reader: L{OVFReader}
999
  @ivar ovf_reader: OVF reader instance collects data from .ovf file
1000
  @type results_name: string
1001
  @ivar results_name: name of imported instance
1002
  @type results_template: string
1003
  @ivar results_template: disk template read from .ovf file or command line
1004
    arguments
1005
  @type results_hypervisor: dict
1006
  @ivar results_hypervisor: hypervisor information gathered from .ovf file or
1007
    command line arguments
1008
  @type results_os: dict
1009
  @ivar results_os: operating system information gathered from .ovf file or
1010
    command line arguments
1011
  @type results_backend: dict
1012
  @ivar results_backend: backend information gathered from .ovf file or
1013
    command line arguments
1014
  @type results_tags: string
1015
  @ivar results_tags: string containing instance-specific tags
1016
  @type results_version: string
1017
  @ivar results_version: version as required by Ganeti import
1018
  @type results_network: dict
1019
  @ivar results_network: network information gathered from .ovf file or command
1020
    line arguments
1021
  @type results_disk: dict
1022
  @ivar results_disk: disk information gathered from .ovf file or command line
1023
    arguments
1024

1025
  """
1026
  def _ReadInputData(self, input_path):
1027
    """Reads the data on which the conversion will take place.
1028

1029
    @type input_path: string
1030
    @param input_path: absolute path to the .ovf or .ova input file
1031

1032
    @raise errors.OpPrereqError: if input file is neither .ovf nor .ova
1033

1034
    """
1035
    (input_dir, input_file) = os.path.split(input_path)
1036
    (_, input_extension) = os.path.splitext(input_file)
1037

    
1038
    if input_extension == OVF_EXT:
1039
      logging.info("%s file extension found, no unpacking necessary", OVF_EXT)
1040
      self.input_dir = input_dir
1041
      self.input_path = input_path
1042
      self.temp_dir = None
1043
    elif input_extension == OVA_EXT:
1044
      logging.info("%s file extension found, proceeding to unpacking", OVA_EXT)
1045
      self._UnpackOVA(input_path)
1046
    else:
1047
      raise errors.OpPrereqError("Unknown file extension; expected %s or %s"
1048
                                 " file" % (OVA_EXT, OVF_EXT))
1049
    assert ((input_extension == OVA_EXT and self.temp_dir) or
1050
            (input_extension == OVF_EXT and not self.temp_dir))
1051
    assert self.input_dir in self.input_path
1052

    
1053
    if self.options.output_dir:
1054
      self.output_dir = os.path.abspath(self.options.output_dir)
1055
      if (os.path.commonprefix([constants.EXPORT_DIR, self.output_dir]) !=
1056
          constants.EXPORT_DIR):
1057
        logging.warning("Export path is not under %s directory, import to"
1058
                        " Ganeti using gnt-backup may fail",
1059
                        constants.EXPORT_DIR)
1060
    else:
1061
      self.output_dir = constants.EXPORT_DIR
1062

    
1063
    self.ovf_reader = OVFReader(self.input_path)
1064
    self.ovf_reader.VerifyManifest()
1065

    
1066
  def _UnpackOVA(self, input_path):
1067
    """Unpacks the .ova package into temporary directory.
1068

1069
    @type input_path: string
1070
    @param input_path: path to the .ova package file
1071

1072
    @raise errors.OpPrereqError: if file is not a proper tarball, one of the
1073
        files in the archive seem malicious (e.g. path starts with '../') or
1074
        .ova package does not contain .ovf file
1075

1076
    """
1077
    input_name = None
1078
    if not tarfile.is_tarfile(input_path):
1079
      raise errors.OpPrereqError("The provided %s file is not a proper tar"
1080
                                 " archive", OVA_EXT)
1081
    ova_content = tarfile.open(input_path)
1082
    temp_dir = tempfile.mkdtemp()
1083
    self.temp_dir = temp_dir
1084
    for file_name in ova_content.getnames():
1085
      file_normname = os.path.normpath(file_name)
1086
      try:
1087
        utils.PathJoin(temp_dir, file_normname)
1088
      except ValueError, err:
1089
        raise errors.OpPrereqError("File %s inside %s package is not safe" %
1090
                                   (file_name, OVA_EXT))
1091
      if file_name.endswith(OVF_EXT):
1092
        input_name = file_name
1093
    if not input_name:
1094
      raise errors.OpPrereqError("No %s file in %s package found" %
1095
                                 (OVF_EXT, OVA_EXT))
1096
    logging.warning("Unpacking the %s archive, this may take a while",
1097
      input_path)
1098
    self.input_dir = temp_dir
1099
    self.input_path = utils.PathJoin(self.temp_dir, input_name)
1100
    try:
1101
      try:
1102
        extract = ova_content.extractall
1103
      except AttributeError:
1104
        # This is a prehistorical case of using python < 2.5
1105
        for member in ova_content.getmembers():
1106
          ova_content.extract(member, path=self.temp_dir)
1107
      else:
1108
        extract(self.temp_dir)
1109
    except tarfile.TarError, err:
1110
      raise errors.OpPrereqError("Error while extracting %s archive: %s" %
1111
                                 (OVA_EXT, err))
1112
    logging.info("OVA package extracted to %s directory", self.temp_dir)
1113

    
1114
  def Parse(self):
1115
    """Parses the data and creates a structure containing all required info.
1116

1117
    The method reads the information given either as a command line option or as
1118
    a part of the OVF description.
1119

1120
    @raise errors.OpPrereqError: if some required part of the description of
1121
      virtual instance is missing or unable to create output directory
1122

1123
    """
1124
    self.results_name = self._GetInfo("instance name", self.options.name,
1125
      self._ParseNameOptions, self.ovf_reader.GetInstanceName)
1126
    if not self.results_name:
1127
      raise errors.OpPrereqError("Name of instance not provided")
1128

    
1129
    self.output_dir = utils.PathJoin(self.output_dir, self.results_name)
1130
    try:
1131
      utils.Makedirs(self.output_dir)
1132
    except OSError, err:
1133
      raise errors.OpPrereqError("Failed to create directory %s: %s" %
1134
                                 (self.output_dir, err))
1135

    
1136
    self.results_template = self._GetInfo("disk template",
1137
      self.options.disk_template, self._ParseTemplateOptions,
1138
      self.ovf_reader.GetDiskTemplate)
1139
    if not self.results_template:
1140
      logging.info("Disk template not given")
1141

    
1142
    self.results_hypervisor = self._GetInfo("hypervisor",
1143
      self.options.hypervisor, self._ParseHypervisorOptions,
1144
      self.ovf_reader.GetHypervisorData)
1145
    assert self.results_hypervisor["hypervisor_name"]
1146
    if self.results_hypervisor["hypervisor_name"] == constants.VALUE_AUTO:
1147
      logging.debug("Default hypervisor settings from the cluster will be used")
1148

    
1149
    self.results_os = self._GetInfo("OS", self.options.os,
1150
      self._ParseOSOptions, self.ovf_reader.GetOSData)
1151
    if not self.results_os.get("os_name"):
1152
      raise errors.OpPrereqError("OS name must be provided")
1153

    
1154
    self.results_backend = self._GetInfo("backend", self.options.beparams,
1155
      self._ParseBackendOptions, self.ovf_reader.GetBackendData)
1156
    assert self.results_backend.get("vcpus")
1157
    assert self.results_backend.get("memory")
1158
    assert self.results_backend.get("auto_balance") is not None
1159

    
1160
    self.results_tags = self._GetInfo("tags", self.options.tags,
1161
      self._ParseTags, self.ovf_reader.GetTagsData)
1162

    
1163
    ovf_version = self.ovf_reader.GetVersionData()
1164
    if ovf_version:
1165
      self.results_version = ovf_version
1166
    else:
1167
      self.results_version = constants.EXPORT_VERSION
1168

    
1169
    self.results_network = self._GetInfo("network", self.options.nics,
1170
      self._ParseNicOptions, self.ovf_reader.GetNetworkData,
1171
      ignore_test=self.options.no_nics)
1172

    
1173
    self.results_disk = self._GetInfo("disk", self.options.disks,
1174
      self._ParseDiskOptions, self._GetDiskInfo,
1175
      ignore_test=self.results_template == constants.DT_DISKLESS)
1176

    
1177
    if not self.results_disk and not self.results_network:
1178
      raise errors.OpPrereqError("Either disk specification or network"
1179
                                 " description must be present")
1180

    
1181
  @staticmethod
1182
  def _GetInfo(name, cmd_arg, cmd_function, nocmd_function,
1183
    ignore_test=False):
1184
    """Get information about some section - e.g. disk, network, hypervisor.
1185

1186
    @type name: string
1187
    @param name: name of the section
1188
    @type cmd_arg: dict
1189
    @param cmd_arg: command line argument specific for section 'name'
1190
    @type cmd_function: callable
1191
    @param cmd_function: function to call if 'cmd_args' exists
1192
    @type nocmd_function: callable
1193
    @param nocmd_function: function to call if 'cmd_args' is not there
1194

1195
    """
1196
    if ignore_test:
1197
      logging.info("Information for %s will be ignored", name)
1198
      return {}
1199
    if cmd_arg:
1200
      logging.info("Information for %s will be parsed from command line", name)
1201
      results = cmd_function()
1202
    else:
1203
      logging.info("Information for %s will be parsed from %s file",
1204
        name, OVF_EXT)
1205
      results = nocmd_function()
1206
    logging.info("Options for %s were succesfully read", name)
1207
    return results
1208

    
1209
  def _ParseNameOptions(self):
1210
    """Returns name if one was given in command line.
1211

1212
    @rtype: string
1213
    @return: name of an instance
1214

1215
    """
1216
    return self.options.name
1217

    
1218
  def _ParseTemplateOptions(self):
1219
    """Returns disk template if one was given in command line.
1220

1221
    @rtype: string
1222
    @return: disk template name
1223

1224
    """
1225
    return self.options.disk_template
1226

    
1227
  def _ParseHypervisorOptions(self):
1228
    """Parses hypervisor options given in a command line.
1229

1230
    @rtype: dict
1231
    @return: dictionary containing name of the chosen hypervisor and all the
1232
      options
1233

1234
    """
1235
    assert type(self.options.hypervisor) is tuple
1236
    assert len(self.options.hypervisor) == 2
1237
    results = {}
1238
    if self.options.hypervisor[0]:
1239
      results["hypervisor_name"] = self.options.hypervisor[0]
1240
    else:
1241
      results["hypervisor_name"] = constants.VALUE_AUTO
1242
    results.update(self.options.hypervisor[1])
1243
    return results
1244

    
1245
  def _ParseOSOptions(self):
1246
    """Parses OS options given in command line.
1247

1248
    @rtype: dict
1249
    @return: dictionary containing name of chosen OS and all its options
1250

1251
    """
1252
    assert self.options.os
1253
    results = {}
1254
    results["os_name"] = self.options.os
1255
    results.update(self.options.osparams)
1256
    return results
1257

    
1258
  def _ParseBackendOptions(self):
1259
    """Parses backend options given in command line.
1260

1261
    @rtype: dict
1262
    @return: dictionary containing vcpus, memory and auto-balance options
1263

1264
    """
1265
    assert self.options.beparams
1266
    backend = {}
1267
    backend.update(self.options.beparams)
1268
    must_contain = ["vcpus", "memory", "auto_balance"]
1269
    for element in must_contain:
1270
      if backend.get(element) is None:
1271
        backend[element] = constants.VALUE_AUTO
1272
    return backend
1273

    
1274
  def _ParseTags(self):
1275
    """Returns tags list given in command line.
1276

1277
    @rtype: string
1278
    @return: string containing comma-separated tags
1279

1280
    """
1281
    return self.options.tags
1282

    
1283
  def _ParseNicOptions(self):
1284
    """Parses network options given in a command line or as a dictionary.
1285

1286
    @rtype: dict
1287
    @return: dictionary of network-related options
1288

1289
    """
1290
    assert self.options.nics
1291
    results = {}
1292
    for (nic_id, nic_desc) in self.options.nics:
1293
      results["nic%s_mode" % nic_id] = \
1294
        nic_desc.get("mode", constants.VALUE_AUTO)
1295
      results["nic%s_mac" % nic_id] = nic_desc.get("mac", constants.VALUE_AUTO)
1296
      results["nic%s_link" % nic_id] = \
1297
        nic_desc.get("link", constants.VALUE_AUTO)
1298
      if nic_desc.get("mode") == "bridged":
1299
        results["nic%s_ip" % nic_id] = constants.VALUE_NONE
1300
      else:
1301
        results["nic%s_ip" % nic_id] = constants.VALUE_AUTO
1302
    results["nic_count"] = str(len(self.options.nics))
1303
    return results
1304

    
1305
  def _ParseDiskOptions(self):
1306
    """Parses disk options given in a command line.
1307

1308
    @rtype: dict
1309
    @return: dictionary of disk-related options
1310

1311
    @raise errors.OpPrereqError: disk description does not contain size
1312
      information or size information is invalid or creation failed
1313

1314
    """
1315
    assert self.options.disks
1316
    results = {}
1317
    for (disk_id, disk_desc) in self.options.disks:
1318
      results["disk%s_ivname" % disk_id] = "disk/%s" % disk_id
1319
      if disk_desc.get("size"):
1320
        try:
1321
          disk_size = utils.ParseUnit(disk_desc["size"])
1322
        except ValueError:
1323
          raise errors.OpPrereqError("Invalid disk size for disk %s: %s" %
1324
                                     (disk_id, disk_desc["size"]))
1325
        new_path = utils.PathJoin(self.output_dir, str(disk_id))
1326
        args = [
1327
          "qemu-img",
1328
          "create",
1329
          "-f",
1330
          "raw",
1331
          new_path,
1332
          disk_size,
1333
        ]
1334
        run_result = utils.RunCmd(args)
1335
        if run_result.failed:
1336
          raise errors.OpPrereqError("Creation of disk %s failed, output was:"
1337
                                     " %s" % (new_path, run_result.stderr))
1338
        results["disk%s_size" % disk_id] = str(disk_size)
1339
        results["disk%s_dump" % disk_id] = "disk%s.raw" % disk_id
1340
      else:
1341
        raise errors.OpPrereqError("Disks created for import must have their"
1342
                                   " size specified")
1343
    results["disk_count"] = str(len(self.options.disks))
1344
    return results
1345

    
1346
  def _GetDiskInfo(self):
1347
    """Gathers information about disks used by instance, perfomes conversion.
1348

1349
    @rtype: dict
1350
    @return: dictionary of disk-related options
1351

1352
    @raise errors.OpPrereqError: disk is not in the same directory as .ovf file
1353

1354
    """
1355
    results = {}
1356
    disks_list = self.ovf_reader.GetDisksNames()
1357
    for (counter, (disk_name, disk_compression)) in enumerate(disks_list):
1358
      if os.path.dirname(disk_name):
1359
        raise errors.OpPrereqError("Disks are not allowed to have absolute"
1360
                                   " paths or paths outside main OVF directory")
1361
      disk, _ = os.path.splitext(disk_name)
1362
      disk_path = utils.PathJoin(self.input_dir, disk_name)
1363
      if disk_compression not in NO_COMPRESSION:
1364
        _, disk_path = self._CompressDisk(disk_path, disk_compression,
1365
          DECOMPRESS)
1366
        disk, _ = os.path.splitext(disk)
1367
      if self._GetDiskQemuInfo(disk_path, "file format: (\S+)") != "raw":
1368
        logging.info("Conversion to raw format is required")
1369
      ext, new_disk_path = self._ConvertDisk("raw", disk_path)
1370

    
1371
      final_disk_path = LinkFile(new_disk_path, prefix=disk, suffix=ext,
1372
        directory=self.output_dir)
1373
      final_name = os.path.basename(final_disk_path)
1374
      disk_size = os.path.getsize(final_disk_path) / (1024 * 1024)
1375
      results["disk%s_dump" % counter] = final_name
1376
      results["disk%s_size" % counter] = str(disk_size)
1377
      results["disk%s_ivname" % counter] = "disk/%s" % str(counter)
1378
    if disks_list:
1379
      results["disk_count"] = str(len(disks_list))
1380
    return results
1381

    
1382
  def Save(self):
1383
    """Saves all the gathered information in a constant.EXPORT_CONF_FILE file.
1384

1385
    @raise errors.OpPrereqError: when saving to config file failed
1386

1387
    """
1388
    logging.info("Conversion was succesfull, saving %s in %s directory",
1389
                 constants.EXPORT_CONF_FILE, self.output_dir)
1390
    results = {
1391
      constants.INISECT_INS: {},
1392
      constants.INISECT_BEP: {},
1393
      constants.INISECT_EXP: {},
1394
      constants.INISECT_OSP: {},
1395
      constants.INISECT_HYP: {},
1396
    }
1397

    
1398
    results[constants.INISECT_INS].update(self.results_disk)
1399
    results[constants.INISECT_INS].update(self.results_network)
1400
    results[constants.INISECT_INS]["hypervisor"] = \
1401
      self.results_hypervisor["hypervisor_name"]
1402
    results[constants.INISECT_INS]["name"] = self.results_name
1403
    if self.results_template:
1404
      results[constants.INISECT_INS]["disk_template"] = self.results_template
1405
    if self.results_tags:
1406
      results[constants.INISECT_INS]["tags"] = self.results_tags
1407

    
1408
    results[constants.INISECT_BEP].update(self.results_backend)
1409

    
1410
    results[constants.INISECT_EXP]["os"] = self.results_os["os_name"]
1411
    results[constants.INISECT_EXP]["version"] = self.results_version
1412

    
1413
    del self.results_os["os_name"]
1414
    results[constants.INISECT_OSP].update(self.results_os)
1415

    
1416
    del self.results_hypervisor["hypervisor_name"]
1417
    results[constants.INISECT_HYP].update(self.results_hypervisor)
1418

    
1419
    output_file_name = utils.PathJoin(self.output_dir,
1420
      constants.EXPORT_CONF_FILE)
1421

    
1422
    output = []
1423
    for section, options in results.iteritems():
1424
      output.append("[%s]" % section)
1425
      for name, value in options.iteritems():
1426
        if value is None:
1427
          value = ""
1428
        output.append("%s = %s" % (name, value))
1429
      output.append("")
1430
    output_contents = "\n".join(output)
1431

    
1432
    try:
1433
      utils.WriteFile(output_file_name, data=output_contents)
1434
    except errors.ProgrammerError, err:
1435
      raise errors.OpPrereqError("Saving the config file failed: %s" % err)
1436

    
1437
    self.Cleanup()
1438

    
1439

    
1440
class ConfigParserWithDefaults(ConfigParser.SafeConfigParser):
1441
  """This is just a wrapper on SafeConfigParser, that uses default values
1442

1443
  """
1444
  def get(self, section, options, raw=None, vars=None): # pylint: disable=W0622
1445
    try:
1446
      result = ConfigParser.SafeConfigParser.get(self, section, options, \
1447
        raw=raw, vars=vars)
1448
    except ConfigParser.NoOptionError:
1449
      result = None
1450
    return result
1451

    
1452
  def getint(self, section, options):
1453
    try:
1454
      result = ConfigParser.SafeConfigParser.get(self, section, options)
1455
    except ConfigParser.NoOptionError:
1456
      result = 0
1457
    return int(result)
1458

    
1459

    
1460
class OVFExporter(Converter):
1461
  """Converter from Ganeti config file to OVF
1462

1463
  @type input_dir: string
1464
  @ivar input_dir: directory in which the config.ini file resides
1465
  @type output_dir: string
1466
  @ivar output_dir: directory to which the results of conversion shall be
1467
    written
1468
  @type packed_dir: string
1469
  @ivar packed_dir: if we want OVA package, this points to the real (i.e. not
1470
    temp) output directory
1471
  @type input_path: string
1472
  @ivar input_path: complete path to the config.ini file
1473
  @type output_path: string
1474
  @ivar output_path: complete path to .ovf file
1475
  @type config_parser: L{ConfigParserWithDefaults}
1476
  @ivar config_parser: parser for the config.ini file
1477
  @type reference_files: list
1478
  @ivar reference_files: files referenced in the ovf file
1479
  @type results_disk: list
1480
  @ivar results_disk: list of dictionaries of disk options from config.ini
1481
  @type results_network: list
1482
  @ivar results_network: list of dictionaries of network options form config.ini
1483
  @type results_name: string
1484
  @ivar results_name: name of the instance
1485
  @type results_vcpus: string
1486
  @ivar results_vcpus: number of VCPUs
1487
  @type results_memory: string
1488
  @ivar results_memory: RAM memory in MB
1489
  @type results_ganeti: dict
1490
  @ivar results_ganeti: dictionary of Ganeti-specific options from config.ini
1491

1492
  """
1493
  def _ReadInputData(self, input_path):
1494
    """Reads the data on which the conversion will take place.
1495

1496
    @type input_path: string
1497
    @param input_path: absolute path to the config.ini input file
1498

1499
    @raise errors.OpPrereqError: error when reading the config file
1500

1501
    """
1502
    input_dir = os.path.dirname(input_path)
1503
    self.input_path = input_path
1504
    self.input_dir = input_dir
1505
    if self.options.output_dir:
1506
      self.output_dir = os.path.abspath(self.options.output_dir)
1507
    else:
1508
      self.output_dir = input_dir
1509
    self.config_parser = ConfigParserWithDefaults()
1510
    logging.info("Reading configuration from %s file", input_path)
1511
    try:
1512
      self.config_parser.read(input_path)
1513
    except ConfigParser.MissingSectionHeaderError, err:
1514
      raise errors.OpPrereqError("Error when trying to read %s: %s" %
1515
                                 (input_path, err))
1516
    if self.options.ova_package:
1517
      self.temp_dir = tempfile.mkdtemp()
1518
      self.packed_dir = self.output_dir
1519
      self.output_dir = self.temp_dir
1520

    
1521
    self.ovf_writer = OVFWriter(not self.options.ext_usage)
1522

    
1523
  def _ParseName(self):
1524
    """Parses name from command line options or config file.
1525

1526
    @rtype: string
1527
    @return: name of Ganeti instance
1528

1529
    @raise errors.OpPrereqError: if name of the instance is not provided
1530

1531
    """
1532
    if self.options.name:
1533
      name = self.options.name
1534
    else:
1535
      name = self.config_parser.get(constants.INISECT_INS, NAME)
1536
    if name is None:
1537
      raise errors.OpPrereqError("No instance name found")
1538
    return name
1539

    
1540
  def _ParseVCPUs(self):
1541
    """Parses vcpus number from config file.
1542

1543
    @rtype: int
1544
    @return: number of virtual CPUs
1545

1546
    @raise errors.OpPrereqError: if number of VCPUs equals 0
1547

1548
    """
1549
    vcpus = self.config_parser.getint(constants.INISECT_BEP, VCPUS)
1550
    if vcpus == 0:
1551
      raise errors.OpPrereqError("No CPU information found")
1552
    return vcpus
1553

    
1554
  def _ParseMemory(self):
1555
    """Parses vcpus number from config file.
1556

1557
    @rtype: int
1558
    @return: amount of memory in MB
1559

1560
    @raise errors.OpPrereqError: if amount of memory equals 0
1561

1562
    """
1563
    memory = self.config_parser.getint(constants.INISECT_BEP, MEMORY)
1564
    if memory == 0:
1565
      raise errors.OpPrereqError("No memory information found")
1566
    return memory
1567

    
1568
  def _ParseGaneti(self):
1569
    """Parses Ganeti data from config file.
1570

1571
    @rtype: dictionary
1572
    @return: dictionary of Ganeti-specific options
1573

1574
    """
1575
    results = {}
1576
    # hypervisor
1577
    results["hypervisor"] = {}
1578
    hyp_name = self.config_parser.get(constants.INISECT_INS, HYPERV)
1579
    if hyp_name is None:
1580
      raise errors.OpPrereqError("No hypervisor information found")
1581
    results["hypervisor"]["name"] = hyp_name
1582
    pairs = self.config_parser.items(constants.INISECT_HYP)
1583
    for (name, value) in pairs:
1584
      results["hypervisor"][name] = value
1585
    # os
1586
    results["os"] = {}
1587
    os_name = self.config_parser.get(constants.INISECT_EXP, OS)
1588
    if os_name is None:
1589
      raise errors.OpPrereqError("No operating system information found")
1590
    results["os"]["name"] = os_name
1591
    pairs = self.config_parser.items(constants.INISECT_OSP)
1592
    for (name, value) in pairs:
1593
      results["os"][name] = value
1594
    # other
1595
    others = [
1596
      (constants.INISECT_INS, DISK_TEMPLATE, "disk_template"),
1597
      (constants.INISECT_BEP, AUTO_BALANCE, "auto_balance"),
1598
      (constants.INISECT_INS, TAGS, "tags"),
1599
      (constants.INISECT_EXP, VERSION, "version"),
1600
    ]
1601
    for (section, element, name) in others:
1602
      results[name] = self.config_parser.get(section, element)
1603
    return results
1604

    
1605
  def _ParseNetworks(self):
1606
    """Parses network data from config file.
1607

1608
    @rtype: list
1609
    @return: list of dictionaries of network options
1610

1611
    @raise errors.OpPrereqError: then network mode is not recognized
1612

1613
    """
1614
    results = []
1615
    counter = 0
1616
    while True:
1617
      data_link = \
1618
        self.config_parser.get(constants.INISECT_INS, "nic%s_link" % counter)
1619
      if data_link is None:
1620
        break
1621
      results.append({
1622
        "mode": self.config_parser.get(constants.INISECT_INS,
1623
           "nic%s_mode" % counter),
1624
        "mac": self.config_parser.get(constants.INISECT_INS,
1625
           "nic%s_mac" % counter),
1626
        "ip": self.config_parser.get(constants.INISECT_INS,
1627
           "nic%s_ip" % counter),
1628
        "link": data_link,
1629
      })
1630
      if results[counter]["mode"] not in constants.NIC_VALID_MODES:
1631
        raise errors.OpPrereqError("Network mode %s not recognized"
1632
                                   % results[counter]["mode"])
1633
      counter += 1
1634
    return results
1635

    
1636
  def _GetDiskOptions(self, disk_file, compression):
1637
    """Convert the disk and gather disk info for .ovf file.
1638

1639
    @type disk_file: string
1640
    @param disk_file: name of the disk (without the full path)
1641
    @type compression: bool
1642
    @param compression: whether the disk should be compressed or not
1643

1644
    @raise errors.OpPrereqError: when disk image does not exist
1645

1646
    """
1647
    disk_path = utils.PathJoin(self.input_dir, disk_file)
1648
    results = {}
1649
    if not os.path.isfile(disk_path):
1650
      raise errors.OpPrereqError("Disk image does not exist: %s" % disk_path)
1651
    if os.path.dirname(disk_file):
1652
      raise errors.OpPrereqError("Path for the disk: %s contains a directory"
1653
                                 " name" % disk_path)
1654
    disk_name, _ = os.path.splitext(disk_file)
1655
    ext, new_disk_path = self._ConvertDisk(self.options.disk_format, disk_path)
1656
    results["format"] = self.options.disk_format
1657
    results["virt-size"] = self._GetDiskQemuInfo(new_disk_path,
1658
      "virtual size: \S+ \((\d+) bytes\)")
1659
    if compression:
1660
      ext2, new_disk_path = self._CompressDisk(new_disk_path, "gzip",
1661
        COMPRESS)
1662
      disk_name, _ = os.path.splitext(disk_name)
1663
      results["compression"] = "gzip"
1664
      ext += ext2
1665
    final_disk_path = LinkFile(new_disk_path, prefix=disk_name, suffix=ext,
1666
      directory=self.output_dir)
1667
    final_disk_name = os.path.basename(final_disk_path)
1668
    results["real-size"] = os.path.getsize(final_disk_path)
1669
    results["path"] = final_disk_name
1670
    self.references_files.append(final_disk_path)
1671
    return results
1672

    
1673
  def _ParseDisks(self):
1674
    """Parses disk data from config file.
1675

1676
    @rtype: list
1677
    @return: list of dictionaries of disk options
1678

1679
    """
1680
    results = []
1681
    counter = 0
1682
    while True:
1683
      disk_file = \
1684
        self.config_parser.get(constants.INISECT_INS, "disk%s_dump" % counter)
1685
      if disk_file is None:
1686
        break
1687
      results.append(self._GetDiskOptions(disk_file, self.options.compression))
1688
      counter += 1
1689
    return results
1690

    
1691
  def Parse(self):
1692
    """Parses the data and creates a structure containing all required info.
1693

1694
    """
1695
    try:
1696
      utils.Makedirs(self.output_dir)
1697
    except OSError, err:
1698
      raise errors.OpPrereqError("Failed to create directory %s: %s" %
1699
                                 (self.output_dir, err))
1700

    
1701
    self.references_files = []
1702
    self.results_name = self._ParseName()
1703
    self.results_vcpus = self._ParseVCPUs()
1704
    self.results_memory = self._ParseMemory()
1705
    if not self.options.ext_usage:
1706
      self.results_ganeti = self._ParseGaneti()
1707
    self.results_network = self._ParseNetworks()
1708
    self.results_disk = self._ParseDisks()
1709

    
1710
  def _PrepareManifest(self, path):
1711
    """Creates manifest for all the files in OVF package.
1712

1713
    @type path: string
1714
    @param path: path to manifesto file
1715

1716
    @raise errors.OpPrereqError: if error occurs when writing file
1717

1718
    """
1719
    logging.info("Preparing manifest for the OVF package")
1720
    lines = []
1721
    files_list = [self.output_path]
1722
    files_list.extend(self.references_files)
1723
    logging.warning("Calculating SHA1 checksums, this may take a while")
1724
    sha1_sums = utils.FingerprintFiles(files_list)
1725
    for file_path, value in sha1_sums.iteritems():
1726
      file_name = os.path.basename(file_path)
1727
      lines.append("SHA1(%s)= %s" % (file_name, value))
1728
    lines.append("")
1729
    data = "\n".join(lines)
1730
    try:
1731
      utils.WriteFile(path, data=data)
1732
    except errors.ProgrammerError, err:
1733
      raise errors.OpPrereqError("Saving the manifest file failed: %s" % err)
1734

    
1735
  @staticmethod
1736
  def _PrepareTarFile(tar_path, files_list):
1737
    """Creates tarfile from the files in OVF package.
1738

1739
    @type tar_path: string
1740
    @param tar_path: path to the resulting file
1741
    @type files_list: list
1742
    @param files_list: list of files in the OVF package
1743

1744
    """
1745
    logging.info("Preparing tarball for the OVF package")
1746
    open(tar_path, mode="w").close()
1747
    ova_package = tarfile.open(name=tar_path, mode="w")
1748
    for file_path in files_list:
1749
      file_name = os.path.basename(file_path)
1750
      ova_package.add(file_path, arcname=file_name)
1751
    ova_package.close()
1752

    
1753
  def Save(self):
1754
    """Saves the gathered configuration in an apropriate format.
1755

1756
    @raise errors.OpPrereqError: if unable to create output directory
1757

1758
    """
1759
    output_file = "%s%s" % (self.results_name, OVF_EXT)
1760
    output_path = utils.PathJoin(self.output_dir, output_file)
1761
    self.ovf_writer = OVFWriter(not self.options.ext_usage)
1762
    logging.info("Saving read data to %s", output_path)
1763

    
1764
    self.output_path = utils.PathJoin(self.output_dir, output_file)
1765
    files_list = [self.output_path]
1766

    
1767
    self.ovf_writer.SaveDisksData(self.results_disk)
1768
    self.ovf_writer.SaveNetworksData(self.results_network)
1769
    if not self.options.ext_usage:
1770
      self.ovf_writer.SaveGanetiData(self.results_ganeti, self.results_network)
1771

    
1772
    self.ovf_writer.SaveVirtualSystemData(self.results_name, self.results_vcpus,
1773
      self.results_memory)
1774

    
1775
    data = self.ovf_writer.PrettyXmlDump()
1776
    utils.WriteFile(self.output_path, data=data)
1777

    
1778
    manifest_file = "%s%s" % (self.results_name, MF_EXT)
1779
    manifest_path = utils.PathJoin(self.output_dir, manifest_file)
1780
    self._PrepareManifest(manifest_path)
1781
    files_list.append(manifest_path)
1782

    
1783
    files_list.extend(self.references_files)
1784

    
1785
    if self.options.ova_package:
1786
      ova_file = "%s%s" % (self.results_name, OVA_EXT)
1787
      packed_path = utils.PathJoin(self.packed_dir, ova_file)
1788
      try:
1789
        utils.Makedirs(self.packed_dir)
1790
      except OSError, err:
1791
        raise errors.OpPrereqError("Failed to create directory %s: %s" %
1792
                                   (self.packed_dir, err))
1793
      self._PrepareTarFile(packed_path, files_list)
1794
    logging.info("Creation of the OVF package was successfull")
1795
    self.Cleanup()