X-Git-Url: https://code.grnet.gr/git/ganeti-local/blobdiff_plain/24b9469df90ec86fa9e2beb08325d3580a026689..383477e9c27967731bbe871d266b309dbe4ac43d:/lib/ovf.py diff --git a/lib/ovf.py b/lib/ovf.py index 20f539f..dc75816 100644 --- a/lib/ovf.py +++ b/lib/ovf.py @@ -1,7 +1,7 @@ #!/usr/bin/python # -# Copyright (C) 2011 Google Inc. +# Copyright (C) 2011, 2012 Google Inc. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -29,6 +29,7 @@ # E1101 makes no sense - pylint assumes that ElementTree object is a tuple +import ConfigParser import errno import logging import os @@ -37,15 +38,22 @@ import re import shutil import tarfile import tempfile +import xml.dom.minidom import xml.parsers.expat try: import xml.etree.ElementTree as ET except ImportError: import elementtree.ElementTree as ET +try: + ParseError = ET.ParseError # pylint: disable=E1103 +except AttributeError: + ParseError = None + from ganeti import constants from ganeti import errors from ganeti import utils +from ganeti import pathutils # Schemas used in OVF format @@ -53,6 +61,9 @@ GANETI_SCHEMA = "http://ganeti" OVF_SCHEMA = "http://schemas.dmtf.org/ovf/envelope/1" RASD_SCHEMA = ("http://schemas.dmtf.org/wbem/wscim/1/cim-schema/2/" "CIM_ResourceAllocationSettingData") +VSSD_SCHEMA = ("http://schemas.dmtf.org/wbem/wscim/1/cim-schema/2/" + "CIM_VirtualSystemSettingData") +XML_SCHEMA = "http://www.w3.org/2001/XMLSchema-instance" # File extensions in OVF package OVA_EXT = ".ova" @@ -67,10 +78,83 @@ FILE_EXTENSIONS = [ ] COMPRESSION_TYPE = "gzip" +NO_COMPRESSION = [None, "identity"] COMPRESS = "compression" DECOMPRESS = "decompression" ALLOWED_ACTIONS = [COMPRESS, DECOMPRESS] +VMDK = "vmdk" +RAW = "raw" +COW = "cow" +ALLOWED_FORMATS = [RAW, COW, VMDK] + +# ResourceType values +RASD_TYPE = { + "vcpus": "3", + "memory": "4", + "scsi-controller": "6", + "ethernet-adapter": "10", + "disk": "17", +} + +SCSI_SUBTYPE = "lsilogic" +VS_TYPE = { + "ganeti": "ganeti-ovf", + "external": "vmx-04", +} + +# AllocationUnits values and conversion +ALLOCATION_UNITS = { + "b": ["bytes", "b"], + "kb": ["kilobytes", "kb", "byte * 2^10", "kibibytes", "kib"], + "mb": ["megabytes", "mb", "byte * 2^20", "mebibytes", "mib"], + "gb": ["gigabytes", "gb", "byte * 2^30", "gibibytes", "gib"], +} +CONVERT_UNITS_TO_MB = { + "b": lambda x: x / (1024 * 1024), + "kb": lambda x: x / 1024, + "mb": lambda x: x, + "gb": lambda x: x * 1024, +} + +# Names of the config fields +NAME = "name" +OS = "os" +HYPERV = "hypervisor" +VCPUS = "vcpus" +MEMORY = "memory" +AUTO_BALANCE = "auto_balance" +DISK_TEMPLATE = "disk_template" +TAGS = "tags" +VERSION = "version" + +# Instance IDs of System and SCSI controller +INSTANCE_ID = { + "system": 0, + "vcpus": 1, + "memory": 2, + "scsi": 3, +} + +# Disk format descriptions +DISK_FORMAT = { + RAW: "http://en.wikipedia.org/wiki/Byte", + VMDK: "http://www.vmware.com/interfaces/specifications/vmdk.html" + "#monolithicSparse", + COW: "http://www.gnome.org/~markmc/qcow-image-format.html", +} + + +def CheckQemuImg(): + """ Make sure that qemu-img is present before performing operations. + + @raise errors.OpPrereqError: when qemu-img was not found in the system + + """ + if not constants.QEMUIMG_PATH: + raise errors.OpPrereqError("qemu-img not found at build time, unable" + " to continue", errors.ECODE_STATE) + def LinkFile(old_path, prefix=None, suffix=None, directory=None): """Create link with a given prefix and suffix. @@ -104,11 +188,12 @@ def LinkFile(old_path, prefix=None, suffix=None, directory=None): except OSError, err: if err.errno == errno.EEXIST: new_path = utils.PathJoin(directory, - "%s_%s%s" % (prefix, counter, suffix)) + "%s_%s%s" % (prefix, counter, suffix)) counter += 1 else: raise errors.OpPrereqError("Error moving the file %s to %s location:" - " %s" % (old_path, new_path, err)) + " %s" % (old_path, new_path, err), + errors.ECODE_ENVIRON) return new_path @@ -143,9 +228,9 @@ class OVFReader(object): self.tree = ET.ElementTree() try: self.tree.parse(input_path) - except xml.parsers.expat.ExpatError, err: + except (ParseError, xml.parsers.expat.ExpatError), err: raise errors.OpPrereqError("Error while reading %s file: %s" % - (OVF_EXT, err)) + (OVF_EXT, err), errors.ECODE_ENVIRON) # Create a list of all files in the OVF package (input_dir, input_file) = os.path.split(input_path) @@ -162,7 +247,8 @@ class OVFReader(object): for file_name in files_list: file_path = utils.PathJoin(input_dir, file_name) if not os.path.exists(file_path): - raise errors.OpPrereqError("File does not exist: %s" % file_path) + raise errors.OpPrereqError("File does not exist: %s" % file_path, + errors.ECODE_ENVIRON) logging.info("Files in the OVF package: %s", " ".join(files_list)) self.files_list = files_list self.input_dir = input_dir @@ -272,12 +358,13 @@ class OVFReader(object): sha1_sum = match.group(2) manifest_files[file_name] = sha1_sum files_with_paths = [utils.PathJoin(self.input_dir, file_name) - for file_name in self.files_list] + for file_name in self.files_list] sha1_sums = utils.FingerprintFiles(files_with_paths) for file_name, value in manifest_files.iteritems(): if sha1_sums.get(utils.PathJoin(self.input_dir, file_name)) != value: raise errors.OpPrereqError("SHA1 checksum of %s does not match the" - " value in manifest file" % file_name) + " value in manifest file" % file_name, + errors.ECODE_ENVIRON) logging.info("SHA1 checksums verified") def GetInstanceName(self): @@ -300,6 +387,116 @@ class OVFReader(object): (GANETI_SCHEMA, GANETI_SCHEMA)) return self.tree.findtext(find_template) + def GetHypervisorData(self): + """Provides hypervisor information - hypervisor name and options. + + @rtype: dict + @return: dictionary containing name of the used hypervisor and all the + specified options + + """ + hypervisor_search = ("{%s}GanetiSection/{%s}Hypervisor" % + (GANETI_SCHEMA, GANETI_SCHEMA)) + hypervisor_data = self.tree.find(hypervisor_search) + if not hypervisor_data: + return {"hypervisor_name": constants.VALUE_AUTO} + results = { + "hypervisor_name": hypervisor_data.findtext("{%s}Name" % GANETI_SCHEMA, + default=constants.VALUE_AUTO), + } + parameters = hypervisor_data.find("{%s}Parameters" % GANETI_SCHEMA) + results.update(self._GetDictParameters(parameters, GANETI_SCHEMA)) + return results + + def GetOSData(self): + """ Provides operating system information - os name and options. + + @rtype: dict + @return: dictionary containing name and options for the chosen OS + + """ + results = {} + os_search = ("{%s}GanetiSection/{%s}OperatingSystem" % + (GANETI_SCHEMA, GANETI_SCHEMA)) + os_data = self.tree.find(os_search) + if os_data: + results["os_name"] = os_data.findtext("{%s}Name" % GANETI_SCHEMA) + parameters = os_data.find("{%s}Parameters" % GANETI_SCHEMA) + results.update(self._GetDictParameters(parameters, GANETI_SCHEMA)) + return results + + def GetBackendData(self): + """ Provides backend information - vcpus, memory, auto balancing options. + + @rtype: dict + @return: dictionary containing options for vcpus, memory and auto balance + settings + + """ + results = {} + + find_vcpus = ("{%s}VirtualSystem/{%s}VirtualHardwareSection/{%s}Item" % + (OVF_SCHEMA, OVF_SCHEMA, OVF_SCHEMA)) + match_vcpus = ("{%s}ResourceType" % RASD_SCHEMA, RASD_TYPE["vcpus"]) + vcpus = self._GetElementMatchingText(find_vcpus, match_vcpus) + if vcpus: + vcpus_count = vcpus.findtext("{%s}VirtualQuantity" % RASD_SCHEMA, + default=constants.VALUE_AUTO) + else: + vcpus_count = constants.VALUE_AUTO + results["vcpus"] = str(vcpus_count) + + find_memory = find_vcpus + match_memory = ("{%s}ResourceType" % RASD_SCHEMA, RASD_TYPE["memory"]) + memory = self._GetElementMatchingText(find_memory, match_memory) + memory_raw = None + if memory: + alloc_units = memory.findtext("{%s}AllocationUnits" % RASD_SCHEMA) + matching_units = [units for units, variants in ALLOCATION_UNITS.items() + if alloc_units.lower() in variants] + if matching_units == []: + raise errors.OpPrereqError("Unit %s for RAM memory unknown" % + alloc_units, errors.ECODE_INVAL) + units = matching_units[0] + memory_raw = int(memory.findtext("{%s}VirtualQuantity" % RASD_SCHEMA, + default=constants.VALUE_AUTO)) + memory_count = CONVERT_UNITS_TO_MB[units](memory_raw) + else: + memory_count = constants.VALUE_AUTO + results["memory"] = str(memory_count) + + find_balance = ("{%s}GanetiSection/{%s}AutoBalance" % + (GANETI_SCHEMA, GANETI_SCHEMA)) + balance = self.tree.findtext(find_balance, default=constants.VALUE_AUTO) + results["auto_balance"] = balance + + return results + + def GetTagsData(self): + """Provides tags information for instance. + + @rtype: string or None + @return: string of comma-separated tags for the instance + + """ + find_tags = "{%s}GanetiSection/{%s}Tags" % (GANETI_SCHEMA, GANETI_SCHEMA) + results = self.tree.findtext(find_tags) + if results: + return results + else: + return None + + def GetVersionData(self): + """Provides version number read from .ovf file + + @rtype: string + @return: string containing the version number + + """ + find_version = ("{%s}GanetiSection/{%s}Version" % + (GANETI_SCHEMA, GANETI_SCHEMA)) + return self.tree.findtext(find_version) + def GetNetworkData(self): """Provides data about the network in the OVF instance. @@ -317,8 +514,8 @@ class OVFReader(object): networks_search = ("{%s}NetworkSection/{%s}Network" % (OVF_SCHEMA, OVF_SCHEMA)) network_names = self._GetAttributes(networks_search, - "{%s}name" % OVF_SCHEMA) - required = ["ip", "mac", "link", "mode"] + "{%s}name" % OVF_SCHEMA) + required = ["ip", "mac", "link", "mode", "network"] for (counter, network_name) in enumerate(network_names): network_search = ("{%s}VirtualSystem/{%s}VirtualHardwareSection/{%s}Item" % (OVF_SCHEMA, OVF_SCHEMA, OVF_SCHEMA)) @@ -328,7 +525,7 @@ class OVFReader(object): ganeti_match = ("{%s}name" % OVF_SCHEMA, network_name) network_data = self._GetElementMatchingText(network_search, network_match) network_ganeti_data = self._GetElementMatchingAttr(ganeti_search, - ganeti_match) + ganeti_match) ganeti_data = {} if network_ganeti_data: @@ -340,6 +537,8 @@ class OVFReader(object): GANETI_SCHEMA) ganeti_data["link"] = network_ganeti_data.findtext("{%s}Link" % GANETI_SCHEMA) + ganeti_data["network"] = network_ganeti_data.findtext("{%s}Net" % + GANETI_SCHEMA) mac_data = None if network_data: mac_data = network_data.findtext("{%s}Address" % RASD_SCHEMA) @@ -386,13 +585,258 @@ class OVFReader(object): disk_elem = self._GetElementMatchingAttr(disk_search, disk_match) if disk_elem is None: raise errors.OpPrereqError("%s file corrupted - disk %s not found in" - " references" % (OVF_EXT, disk)) + " references" % (OVF_EXT, disk), + errors.ECODE_ENVIRON) disk_name = disk_elem.get("{%s}href" % OVF_SCHEMA) disk_compression = disk_elem.get("{%s}compression" % OVF_SCHEMA) results.append((disk_name, disk_compression)) return results +def SubElementText(parent, tag, text, attrib={}, **extra): +# pylint: disable=W0102 + """This is just a wrapper on ET.SubElement that always has text content. + + """ + if text is None: + return None + elem = ET.SubElement(parent, tag, attrib=attrib, **extra) + elem.text = str(text) + return elem + + +class OVFWriter(object): + """Writer class for OVF files. + + @type tree: ET.ElementTree + @ivar tree: XML tree that we are constructing + @type virtual_system_type: string + @ivar virtual_system_type: value of vssd:VirtualSystemType, for external usage + in VMWare this requires to be vmx + @type hardware_list: list + @ivar hardware_list: list of items prepared for VirtualHardwareSection + @type next_instance_id: int + @ivar next_instance_id: next instance id to be used when creating elements on + hardware_list + + """ + def __init__(self, has_gnt_section): + """Initialize the writer - set the top element. + + @type has_gnt_section: bool + @param has_gnt_section: if the Ganeti schema should be added - i.e. this + means that Ganeti section will be present + + """ + env_attribs = { + "xmlns:xsi": XML_SCHEMA, + "xmlns:vssd": VSSD_SCHEMA, + "xmlns:rasd": RASD_SCHEMA, + "xmlns:ovf": OVF_SCHEMA, + "xmlns": OVF_SCHEMA, + "xml:lang": "en-US", + } + if has_gnt_section: + env_attribs["xmlns:gnt"] = GANETI_SCHEMA + self.virtual_system_type = VS_TYPE["ganeti"] + else: + self.virtual_system_type = VS_TYPE["external"] + self.tree = ET.Element("Envelope", attrib=env_attribs) + self.hardware_list = [] + # INSTANCE_ID contains statically assigned IDs, starting from 0 + self.next_instance_id = len(INSTANCE_ID) # FIXME: hackish + + def SaveDisksData(self, disks): + """Convert disk information to certain OVF sections. + + @type disks: list + @param disks: list of dictionaries of disk options from config.ini + + """ + references = ET.SubElement(self.tree, "References") + disk_section = ET.SubElement(self.tree, "DiskSection") + SubElementText(disk_section, "Info", "Virtual disk information") + for counter, disk in enumerate(disks): + file_id = "file%s" % counter + disk_id = "disk%s" % counter + file_attribs = { + "ovf:href": disk["path"], + "ovf:size": str(disk["real-size"]), + "ovf:id": file_id, + } + disk_attribs = { + "ovf:capacity": str(disk["virt-size"]), + "ovf:diskId": disk_id, + "ovf:fileRef": file_id, + "ovf:format": DISK_FORMAT.get(disk["format"], disk["format"]), + } + if "compression" in disk: + file_attribs["ovf:compression"] = disk["compression"] + ET.SubElement(references, "File", attrib=file_attribs) + ET.SubElement(disk_section, "Disk", attrib=disk_attribs) + + # Item in VirtualHardwareSection creation + disk_item = ET.Element("Item") + SubElementText(disk_item, "rasd:ElementName", disk_id) + SubElementText(disk_item, "rasd:HostResource", "ovf:/disk/%s" % disk_id) + SubElementText(disk_item, "rasd:InstanceID", self.next_instance_id) + SubElementText(disk_item, "rasd:Parent", INSTANCE_ID["scsi"]) + SubElementText(disk_item, "rasd:ResourceType", RASD_TYPE["disk"]) + self.hardware_list.append(disk_item) + self.next_instance_id += 1 + + def SaveNetworksData(self, networks): + """Convert network information to NetworkSection. + + @type networks: list + @param networks: list of dictionaries of network options form config.ini + + """ + network_section = ET.SubElement(self.tree, "NetworkSection") + SubElementText(network_section, "Info", "List of logical networks") + for counter, network in enumerate(networks): + network_name = "%s%s" % (network["mode"], counter) + network_attrib = {"ovf:name": network_name} + ET.SubElement(network_section, "Network", attrib=network_attrib) + + # Item in VirtualHardwareSection creation + network_item = ET.Element("Item") + SubElementText(network_item, "rasd:Address", network["mac"]) + SubElementText(network_item, "rasd:Connection", network_name) + SubElementText(network_item, "rasd:ElementName", network_name) + SubElementText(network_item, "rasd:InstanceID", self.next_instance_id) + SubElementText(network_item, "rasd:ResourceType", + RASD_TYPE["ethernet-adapter"]) + self.hardware_list.append(network_item) + self.next_instance_id += 1 + + @staticmethod + def _SaveNameAndParams(root, data): + """Save name and parameters information under root using data. + + @type root: ET.Element + @param root: root element for the Name and Parameters + @type data: dict + @param data: data from which we gather the values + + """ + assert(data.get("name")) + name = SubElementText(root, "gnt:Name", data["name"]) + params = ET.SubElement(root, "gnt:Parameters") + for name, value in data.iteritems(): + if name != "name": + SubElementText(params, "gnt:%s" % name, value) + + def SaveGanetiData(self, ganeti, networks): + """Convert Ganeti-specific information to GanetiSection. + + @type ganeti: dict + @param ganeti: dictionary of Ganeti-specific options from config.ini + @type networks: list + @param networks: list of dictionaries of network options form config.ini + + """ + ganeti_section = ET.SubElement(self.tree, "gnt:GanetiSection") + + SubElementText(ganeti_section, "gnt:Version", ganeti.get("version")) + SubElementText(ganeti_section, "gnt:DiskTemplate", + ganeti.get("disk_template")) + SubElementText(ganeti_section, "gnt:AutoBalance", + ganeti.get("auto_balance")) + SubElementText(ganeti_section, "gnt:Tags", ganeti.get("tags")) + + osys = ET.SubElement(ganeti_section, "gnt:OperatingSystem") + self._SaveNameAndParams(osys, ganeti["os"]) + + hypervisor = ET.SubElement(ganeti_section, "gnt:Hypervisor") + self._SaveNameAndParams(hypervisor, ganeti["hypervisor"]) + + network_section = ET.SubElement(ganeti_section, "gnt:Network") + for counter, network in enumerate(networks): + network_name = "%s%s" % (network["mode"], counter) + nic_attrib = {"ovf:name": network_name} + nic = ET.SubElement(network_section, "gnt:Nic", attrib=nic_attrib) + SubElementText(nic, "gnt:Mode", network["mode"]) + SubElementText(nic, "gnt:MACAddress", network["mac"]) + SubElementText(nic, "gnt:IPAddress", network["ip"]) + SubElementText(nic, "gnt:Link", network["link"]) + SubElementText(nic, "gnt:Net", network["network"]) + + def SaveVirtualSystemData(self, name, vcpus, memory): + """Convert virtual system information to OVF sections. + + @type name: string + @param name: name of the instance + @type vcpus: int + @param vcpus: number of VCPUs + @type memory: int + @param memory: RAM memory in MB + + """ + assert(vcpus > 0) + assert(memory > 0) + vs_attrib = {"ovf:id": name} + virtual_system = ET.SubElement(self.tree, "VirtualSystem", attrib=vs_attrib) + SubElementText(virtual_system, "Info", "A virtual machine") + + name_section = ET.SubElement(virtual_system, "Name") + name_section.text = name + os_attrib = {"ovf:id": "0"} + os_section = ET.SubElement(virtual_system, "OperatingSystemSection", + attrib=os_attrib) + SubElementText(os_section, "Info", "Installed guest operating system") + hardware_section = ET.SubElement(virtual_system, "VirtualHardwareSection") + SubElementText(hardware_section, "Info", "Virtual hardware requirements") + + # System description + system = ET.SubElement(hardware_section, "System") + SubElementText(system, "vssd:ElementName", "Virtual Hardware Family") + SubElementText(system, "vssd:InstanceID", INSTANCE_ID["system"]) + SubElementText(system, "vssd:VirtualSystemIdentifier", name) + SubElementText(system, "vssd:VirtualSystemType", self.virtual_system_type) + + # Item for vcpus + vcpus_item = ET.SubElement(hardware_section, "Item") + SubElementText(vcpus_item, "rasd:ElementName", + "%s virtual CPU(s)" % vcpus) + SubElementText(vcpus_item, "rasd:InstanceID", INSTANCE_ID["vcpus"]) + SubElementText(vcpus_item, "rasd:ResourceType", RASD_TYPE["vcpus"]) + SubElementText(vcpus_item, "rasd:VirtualQuantity", vcpus) + + # Item for memory + memory_item = ET.SubElement(hardware_section, "Item") + SubElementText(memory_item, "rasd:AllocationUnits", "byte * 2^20") + SubElementText(memory_item, "rasd:ElementName", "%sMB of memory" % memory) + SubElementText(memory_item, "rasd:InstanceID", INSTANCE_ID["memory"]) + SubElementText(memory_item, "rasd:ResourceType", RASD_TYPE["memory"]) + SubElementText(memory_item, "rasd:VirtualQuantity", memory) + + # Item for scsi controller + scsi_item = ET.SubElement(hardware_section, "Item") + SubElementText(scsi_item, "rasd:Address", INSTANCE_ID["system"]) + SubElementText(scsi_item, "rasd:ElementName", "scsi_controller0") + SubElementText(scsi_item, "rasd:InstanceID", INSTANCE_ID["scsi"]) + SubElementText(scsi_item, "rasd:ResourceSubType", SCSI_SUBTYPE) + SubElementText(scsi_item, "rasd:ResourceType", RASD_TYPE["scsi-controller"]) + + # Other items - from self.hardware_list + for item in self.hardware_list: + hardware_section.append(item) + + def PrettyXmlDump(self): + """Formatter of the XML file. + + @rtype: string + @return: XML tree in the form of nicely-formatted string + + """ + raw_string = ET.tostring(self.tree) + parsed_xml = xml.dom.minidom.parseString(raw_string) + xml_string = parsed_xml.toprettyxml(indent=" ") + text_re = re.compile(">\n\s+([^<>\s].*?)\n\s+\g<1>