From f5174d2c356ba6197877b36d04c2dc175d66c7a3 Mon Sep 17 00:00:00 2001 From: Nikos Skalkotos Date: Tue, 26 Mar 2013 18:16:30 +0200 Subject: [PATCH] Rename DiskDevice class to Image Rename DiskDevice class to Image and move it to a seperate module. Also, don't let the user access the os_type/* modules directly. In Image class create the `os' member variable that will host an appropriate instance of one of the OSBase classes. --- image_creator/dialog_main.py | 26 +-- image_creator/dialog_menu.py | 51 +++--- image_creator/dialog_util.py | 16 +- image_creator/dialog_wizard.py | 25 ++- image_creator/disk.py | 341 ++---------------------------------- image_creator/image.py | 372 ++++++++++++++++++++++++++++++++++++++++ image_creator/main.py | 53 +++--- 7 files changed, 464 insertions(+), 420 deletions(-) create mode 100644 image_creator/image.py diff --git a/image_creator/dialog_main.py b/image_creator/dialog_main.py index 9f4d7e8..205ec8c 100644 --- a/image_creator/dialog_main.py +++ b/image_creator/dialog_main.py @@ -48,14 +48,13 @@ from image_creator.output.cli import SimpleOutput from image_creator.output.dialog import GaugeOutput from image_creator.output.composite import CompositeOutput from image_creator.disk import Disk -from image_creator.os_type import os_cls from image_creator.dialog_wizard import wizard from image_creator.dialog_menu import main_menu from image_creator.dialog_util import SMALL_WIDTH, WIDTH, confirm_exit, \ Reset, update_background_title -def image_creator(d, media, out, tmp): +def create_image(d, media, out, tmp): d.setBackgroundTitle('snf-image-creator') @@ -71,19 +70,14 @@ def image_creator(d, media, out, tmp): signal.signal(signal.SIGTERM, signal_handler) try: snapshot = disk.snapshot() - dev = disk.get_device(snapshot) + image = disk.get_image(snapshot) + out.output("Collecting image metadata...") metadata = {} - for (key, value) in dev.meta.items(): + for (key, value) in image.meta.items(): metadata[str(key)] = str(value) - dev.mount(readonly=True) - out.output("Collecting image metadata...") - cls = os_cls(dev.distro, dev.ostype) - image_os = cls(dev.root, dev.g, out) - dev.umount() - - for (key, value) in image_os.meta.items(): + for (key, value) in image.os.meta.items(): metadata[str(key)] = str(value) out.success("done") @@ -97,9 +91,7 @@ def image_creator(d, media, out, tmp): session = {"dialog": d, "disk": disk, - "snapshot": snapshot, - "device": dev, - "image_os": image_os, + "image": image, "metadata": metadata} msg = "snf-image-creator detected a %s system on the input media. " \ @@ -107,8 +99,8 @@ def image_creator(d, media, out, tmp): "image creation process?\n\nChoose to run the wizard," \ " to run the snf-image-creator in expert mode or " \ "press ESC to quit the program." \ - % (dev.ostype if dev.ostype == dev.distro else "%s (%s)" % - (dev.ostype, dev.distro)) + % (image.ostype if image.ostype == image.distro else "%s (%s)" % + (image.ostype, image.distro)) update_background_title(session) @@ -226,7 +218,7 @@ def main(): out = CompositeOutput([log]) out.output("Starting %s v%s..." % (parser.get_prog_name(), version)) - ret = image_creator(d, media, out, options.tmp) + ret = create_image(d, media, out, options.tmp) sys.exit(ret) except Reset: log.output("Resetting everything...") diff --git a/image_creator/dialog_menu.py b/image_creator/dialog_menu.py index 07fa205..3524969 100644 --- a/image_creator/dialog_menu.py +++ b/image_creator/dialog_menu.py @@ -108,9 +108,9 @@ class MetadataMonitor(object): def upload_image(session): d = session["dialog"] - dev = session['device'] + image = session['image'] meta = session['metadata'] - size = dev.size + size = image.size if "account" not in session: d.msgbox("You need to provide your ~okeanos credentials before you " @@ -139,17 +139,17 @@ def upload_image(session): gauge = GaugeOutput(d, "Image Upload", "Uploading...") try: - out = dev.out + out = image.out out.add(gauge) try: if 'checksum' not in session: md5 = MD5(out) - session['checksum'] = md5.compute(session['snapshot'], size) + session['checksum'] = md5.compute(image.device, size) kamaki = Kamaki(session['account'], out) try: # Upload image file - with open(session['snapshot'], 'rb') as f: + with open(image.device, 'rb') as f: session["pithos_uri"] = \ kamaki.upload(f, size, filename, "Calculating block hashes", @@ -188,14 +188,12 @@ def upload_image(session): def register_image(session): d = session["dialog"] - dev = session['device'] is_public = False if "account" not in session: d.msgbox("You need to provide your ~okeanos credentians before you " - "can register an images with cyclades", - width=SMALL_WIDTH) + "can register an images with cyclades", width=SMALL_WIDTH) return False if "pithos_uri" not in session: @@ -233,7 +231,7 @@ def register_image(session): img_type = "public" if is_public else "private" gauge = GaugeOutput(d, "Image Registration", "Registering image...") try: - out = dev.out + out = session['image'].out out.add(gauge) try: out.output("Registering %s image with Cyclades..." % img_type) @@ -481,7 +479,7 @@ def exclude_tasks(session): def sysprep(session): d = session['dialog'] - image_os = session['image_os'] + image = session['image'] # Is the image already shrinked? if 'shrinked' in session and session['shrinked']: @@ -500,7 +498,7 @@ def sysprep(session): if 'exec_syspreps' not in session: session['exec_syspreps'] = [] - all_syspreps = image_os.list_syspreps() + all_syspreps = image.os.list_syspreps() # Only give the user the choice between syspreps that have not ran yet syspreps = [s for s in all_syspreps if s not in session['exec_syspreps']] @@ -513,7 +511,7 @@ def sysprep(session): choices = [] index = 0 for sysprep in syspreps: - name, descr = image_os.sysprep_info(sysprep) + name, descr = image.os.sysprep_info(sysprep) display_name = name.replace('-', ' ').capitalize() sysprep_help += "%s\n" % display_name sysprep_help += "%s\n" % ('-' * len(display_name)) @@ -536,34 +534,33 @@ def sysprep(session): # Enable selected syspreps and disable the rest for i in range(len(syspreps)): if str(i + 1) in tags: - image_os.enable_sysprep(syspreps[i]) + image.os.enable_sysprep(syspreps[i]) session['exec_syspreps'].append(syspreps[i]) else: - image_os.disable_sysprep(syspreps[i]) + image.os.disable_sysprep(syspreps[i]) infobox = InfoBoxOutput(d, "Image Configuration") try: - dev = session['device'] - dev.out.add(infobox) + image.out.add(infobox) try: - dev.mount(readonly=False) + image.mount(readonly=False) try: # The checksum is invalid. We have mounted the image rw if 'checksum' in session: del session['checksum'] # Monitor the metadata changes during syspreps - with MetadataMonitor(session, image_os.meta): - image_os.do_sysprep() + with MetadataMonitor(session, image.os.meta): + image.os.do_sysprep() infobox.finalize() # Disable syspreps that have ran for sysprep in session['exec_syspreps']: - image_os.disable_sysprep(sysprep) + image.os.disable_sysprep(sysprep) finally: - dev.umount() + image.umount() finally: - dev.out.remove(infobox) + image.out.remove(infobox) finally: infobox.cleanup() break @@ -572,7 +569,7 @@ def sysprep(session): def shrink(session): d = session['dialog'] - dev = session['device'] + image = session['image'] shrinked = 'shrinked' in session and session['shrinked'] @@ -589,14 +586,14 @@ def shrink(session): if not d.yesno("%s\n\nDo you want to continue?" % msg, width=WIDTH, height=12, title="Image Shrinking"): - with MetadataMonitor(session, dev.meta): + with MetadataMonitor(session, image.meta): infobox = InfoBoxOutput(d, "Image Shrinking", height=4) - dev.out.add(infobox) + image.out.add(infobox) try: - dev.shrink() + image.shrink() infobox.finalize() finally: - dev.out.remove(infobox) + image.out.remove(infobox) session['shrinked'] = True update_background_title(session) diff --git a/image_creator/dialog_util.py b/image_creator/dialog_util.py index 384019a..f7add02 100644 --- a/image_creator/dialog_util.py +++ b/image_creator/dialog_util.py @@ -43,17 +43,17 @@ WIDTH = 70 def update_background_title(session): d = session['dialog'] - dev = session['device'] disk = session['disk'] + image = session['image'] MB = 2 ** 20 - size = (dev.size + MB - 1) // MB + size = (image.size + MB - 1) // MB shrinked = 'shrinked' in session and session['shrinked'] postfix = " (shrinked)" if shrinked else '' title = "OS: %s, Distro: %s, Size: %dMB%s, Source: %s" % \ - (dev.ostype, dev.distro, size, postfix, + (image.ostype, image.distro, size, postfix, os.path.abspath(disk.source)) d.setBackgroundTitle(title) @@ -128,18 +128,16 @@ def extract_image(session): gauge = GaugeOutput(d, "Image Extraction", "Extracting image...") try: - dev = session['device'] - out = dev.out + image = session['image'] + out = image.out out.add(gauge) try: if "checksum" not in session: - size = dev.size md5 = MD5(out) - session['checksum'] = md5.compute(session['snapshot'], - size) + session['checksum'] = md5.compute(image.device, image.size) # Extract image file - dev.dump(path) + image.dump(path) # Extract metadata file out.output("Extracting metadata file...") diff --git a/image_creator/dialog_wizard.py b/image_creator/dialog_wizard.py index 31aa4a2..e1719cc 100644 --- a/image_creator/dialog_wizard.py +++ b/image_creator/dialog_wizard.py @@ -180,7 +180,7 @@ def wizard(session): name = WizardInputPage( "ImageName", "Image Name", "Please provide a name for the image:", - title="Image Name", init=session['device'].distro) + title="Image Name", init=session['image'].distro) descr = WizardInputPage( "ImageDescription", "Image Description", @@ -232,38 +232,35 @@ def wizard(session): def create_image(session): d = session['dialog'] - disk = session['disk'] - device = session['device'] - snapshot = session['snapshot'] - image_os = session['image_os'] + image = session['image'] wizard = session['wizard'] # Save Kamaki credentials Kamaki.save_token(wizard['Account']['auth_token']) with_progress = OutputWthProgress(True) - out = disk.out + out = image.out out.add(with_progress) try: out.clear() #Sysprep - device.mount(False) - image_os.do_sysprep() - metadata = image_os.meta - device.umount() + image.mount(False) + image.os.do_sysprep() + metadata = image.os.meta + image.umount() #Shrink - size = device.shrink() + size = image.shrink() session['shrinked'] = True update_background_title(session) - metadata.update(device.meta) + metadata.update(image.meta) metadata['DESCRIPTION'] = wizard['ImageDescription'] #MD5 md5 = MD5(out) - session['checksum'] = md5.compute(snapshot, size) + session['checksum'] = md5.compute(image.device, size) #Metadata metastring = '\n'.join( @@ -278,7 +275,7 @@ def create_image(session): name = "%s-%s.diskdump" % (wizard['ImageName'], time.strftime("%Y%m%d%H%M")) pithos_file = "" - with open(snapshot, 'rb') as f: + with open(image.device, 'rb') as f: pithos_file = kamaki.upload(f, size, name, "(1/4) Calculating block hashes", "(2/4) Uploading missing blocks") diff --git a/image_creator/disk.py b/image_creator/disk.py index 9017d45..0645496 100644 --- a/image_creator/disk.py +++ b/image_creator/disk.py @@ -32,21 +32,17 @@ # or implied, of GRNET S.A. from image_creator.util import get_command -from image_creator.util import FatalError from image_creator.util import try_fail_repeat from image_creator.util import free_space -from image_creator.gpt import GPTPartitionTable +from image_creator.util import FatalError from image_creator.bundle_volume import BundleVolume +from image_creator.image import Image import stat import os import tempfile import uuid -import re -import guestfs import shutil -from sendfile import sendfile - dd = get_command('dd') dmsetup = get_command('dmsetup') @@ -70,7 +66,7 @@ class Disk(object): media can be an image file, a block device or a directory. """ self._cleanup_jobs = [] - self._devices = [] + self._images = [] self.source = source self.out = output self.meta = {} @@ -123,9 +119,9 @@ class Disk(object): program ends. """ try: - while len(self._devices): - device = self._devices.pop() - device.destroy() + while len(self._images): + image = self._images.pop() + image.destroy() finally: # Make sure those are executed even if one of the device.destroy # methods throws exeptions. @@ -176,322 +172,19 @@ class Disk(object): self.out.success('done') return "/dev/mapper/%s" % snapshot - def get_device(self, media): - """Returns a newly created DiskDevice instance.""" + def get_image(self, media): + """Returns a newly created ImageCreator instance.""" - new_device = DiskDevice(media, self.out) - self._devices.append(new_device) - new_device.enable() - return new_device + image = Image(media, self.out) + self._images.append(image) + image.enable() + return image - def destroy_device(self, device): - """Destroys a DiskDevice instance previously created by - get_device method. + def destroy_image(self, image): + """Destroys an ImageCreator instance previously created by + get_image_creator method. """ - self._devices.remove(device) - device.destroy() - - -class DiskDevice(object): - """This class represents a block device hosting an Operating System - as created by the device-mapper. - """ - - def __init__(self, device, output, bootable=True, meta={}): - """Create a new DiskDevice.""" - - self.real_device = device - self.out = output - self.bootable = bootable - self.meta = meta - self.progress_bar = None - self.guestfs_device = None - self.size = 0 - - self.g = guestfs.GuestFS() - self.g.add_drive_opts(self.real_device, readonly=0, format="raw") - - # Before version 1.17.14 the recovery process, which is a fork of the - # original process that called libguestfs, did not close its inherited - # file descriptors. This can cause problems especially if the parent - # process has opened pipes. Since the recovery process is an optional - # feature of libguestfs, it's better to disable it. - self.g.set_recovery_proc(0) - version = self.g.version() - if version['major'] > 1 or \ - (version['major'] == 1 and (version['minor'] >= 18 or - (version['minor'] == 17 and - version['release'] >= 14))): - self.g.set_recovery_proc(1) - self.out.output("Enabling recovery proc") - - #self.g.set_trace(1) - #self.g.set_verbose(1) - - self.guestfs_enabled = False - - def enable(self): - """Enable a newly created DiskDevice""" - - self.out.output('Launching helper VM (may take a while) ...', False) - # self.progressbar = self.out.Progress(100, "Launching helper VM", - # "percent") - # eh = self.g.set_event_callback(self.progress_callback, - # guestfs.EVENT_PROGRESS) - self.g.launch() - self.guestfs_enabled = True - # self.g.delete_event_callback(eh) - # self.progressbar.success('done') - # self.progressbar = None - self.out.success('done') - - self.out.output('Inspecting Operating System ...', False) - roots = self.g.inspect_os() - if len(roots) == 0: - raise FatalError("No operating system found") - if len(roots) > 1: - raise FatalError("Multiple operating systems found." - "We only support images with one OS.") - self.root = roots[0] - self.guestfs_device = self.g.part_to_dev(self.root) - self.size = self.g.blockdev_getsize64(self.guestfs_device) - self.meta['PARTITION_TABLE'] = \ - self.g.part_get_parttype(self.guestfs_device) - - self.ostype = self.g.inspect_get_type(self.root) - self.distro = self.g.inspect_get_distro(self.root) - self.out.success('found a(n) %s system' % self.distro) - - def destroy(self): - """Destroy this DiskDevice instance.""" - - # In new guestfs versions, there is a handy shutdown method for this - try: - if self.guestfs_enabled: - self.g.umount_all() - self.g.sync() - finally: - # Close the guestfs handler if open - self.g.close() - -# def progress_callback(self, ev, eh, buf, array): -# position = array[2] -# total = array[3] -# -# self.progressbar.goto((position * 100) // total) - - def mount(self, readonly=False): - """Mount all disk partitions in a correct order.""" - - mount = self.g.mount_ro if readonly else self.g.mount - msg = " read-only" if readonly else "" - self.out.output("Mounting the media%s ..." % msg, False) - mps = self.g.inspect_get_mountpoints(self.root) - - # Sort the keys to mount the fs in a correct order. - # / should be mounted befor /boot, etc - def compare(a, b): - if len(a[0]) > len(b[0]): - return 1 - elif len(a[0]) == len(b[0]): - return 0 - else: - return -1 - mps.sort(compare) - for mp, dev in mps: - try: - mount(dev, mp) - except RuntimeError as msg: - self.out.warn("%s (ignored)" % msg) - self.out.success("done") - - def umount(self): - """Umount all mounted filesystems.""" - self.g.umount_all() - - def _last_partition(self): - if self.meta['PARTITION_TABLE'] not in 'msdos' 'gpt': - msg = "Unsupported partition table: %s. Only msdos and gpt " \ - "partition tables are supported" % self.meta['PARTITION_TABLE'] - raise FatalError(msg) - - is_extended = lambda p: \ - self.g.part_get_mbr_id(self.guestfs_device, p['part_num']) \ - in (0x5, 0xf) - is_logical = lambda p: \ - self.meta['PARTITION_TABLE'] == 'msdos' and p['part_num'] > 4 - - partitions = self.g.part_list(self.guestfs_device) - last_partition = partitions[-1] - - if is_logical(last_partition): - # The disk contains extended and logical partitions.... - extended = filter(is_extended, partitions)[0] - last_primary = [p for p in partitions if p['part_num'] <= 4][-1] - - # check if extended is the last primary partition - if last_primary['part_num'] > extended['part_num']: - last_partition = last_primary - - return last_partition - - def shrink(self): - """Shrink the disk. - - This is accomplished by shrinking the last filesystem in the - disk and then updating the partition table. The new disk size - (in bytes) is returned. - - ATTENTION: make sure unmount is called before shrink - """ - get_fstype = lambda p: \ - self.g.vfs_type("%s%d" % (self.guestfs_device, p['part_num'])) - is_logical = lambda p: \ - self.meta['PARTITION_TABLE'] == 'msdos' and p['part_num'] > 4 - is_extended = lambda p: \ - self.meta['PARTITION_TABLE'] == 'msdos' and \ - self.g.part_get_mbr_id(self.guestfs_device, p['part_num']) \ - in (0x5, 0xf) - - part_add = lambda ptype, start, stop: \ - self.g.part_add(self.guestfs_device, ptype, start, stop) - part_del = lambda p: self.g.part_del(self.guestfs_device, p) - part_get_id = lambda p: self.g.part_get_mbr_id(self.guestfs_device, p) - part_set_id = lambda p, id: \ - self.g.part_set_mbr_id(self.guestfs_device, p, id) - part_get_bootable = lambda p: \ - self.g.part_get_bootable(self.guestfs_device, p) - part_set_bootable = lambda p, bootable: \ - self.g.part_set_bootable(self.guestfs_device, p, bootable) - - MB = 2 ** 20 - - self.out.output("Shrinking image (this may take a while) ...", False) - - sector_size = self.g.blockdev_getss(self.guestfs_device) - - last_part = None - fstype = None - while True: - last_part = self._last_partition() - fstype = get_fstype(last_part) - - if fstype == 'swap': - self.meta['SWAP'] = "%d:%s" % \ - (last_part['part_num'], - (last_part['part_size'] + MB - 1) // MB) - part_del(last_part['part_num']) - continue - elif is_extended(last_part): - part_del(last_part['part_num']) - continue - - # Most disk manipulation programs leave 2048 sectors after the last - # partition - new_size = last_part['part_end'] + 1 + 2048 * sector_size - self.size = min(self.size, new_size) - break - - if not re.match("ext[234]", fstype): - self.out.warn("Don't know how to resize %s partitions." % fstype) - return self.size - - part_dev = "%s%d" % (self.guestfs_device, last_part['part_num']) - self.g.e2fsck_f(part_dev) - self.g.resize2fs_M(part_dev) - - out = self.g.tune2fs_l(part_dev) - block_size = int(filter(lambda x: x[0] == 'Block size', out)[0][1]) - block_cnt = int(filter(lambda x: x[0] == 'Block count', out)[0][1]) - - start = last_part['part_start'] / sector_size - end = start + (block_size * block_cnt) / sector_size - 1 - - if is_logical(last_part): - partitions = self.g.part_list(self.guestfs_device) - - logical = [] # logical partitions - for partition in partitions: - if partition['part_num'] < 4: - continue - logical.append({ - 'num': partition['part_num'], - 'start': partition['part_start'] / sector_size, - 'end': partition['part_end'] / sector_size, - 'id': part_get_id(partition['part_num']), - 'bootable': part_get_bootable(partition['part_num']) - }) - - logical[-1]['end'] = end # new end after resize - - # Recreate the extended partition - extended = filter(is_extended, partitions)[0] - part_del(extended['part_num']) - part_add('e', extended['part_start'] / sector_size, end) - - # Create all the logical partitions back - for l in logical: - part_add('l', l['start'], l['end']) - part_set_id(l['num'], l['id']) - part_set_bootable(l['num'], l['bootable']) - else: - # Recreate the last partition - if self.meta['PARTITION_TABLE'] == 'msdos': - last_part['id'] = part_get_id(last_part['part_num']) - - last_part['bootable'] = part_get_bootable(last_part['part_num']) - part_del(last_part['part_num']) - part_add('p', start, end) - part_set_bootable(last_part['part_num'], last_part['bootable']) - - if self.meta['PARTITION_TABLE'] == 'msdos': - part_set_id(last_part['part_num'], last_part['id']) - - new_size = (end + 1) * sector_size - - assert (new_size <= self.size) - - if self.meta['PARTITION_TABLE'] == 'gpt': - ptable = GPTPartitionTable(self.real_device) - self.size = ptable.shrink(new_size, self.size) - else: - self.size = min(new_size + 2048 * sector_size, self.size) - - self.out.success("new size is %dMB" % ((self.size + MB - 1) // MB)) - - return self.size - - def dump(self, outfile): - """Dumps the content of device into a file. - - This method will only dump the actual payload, found by reading the - partition table. Empty space in the end of the device will be ignored. - """ - MB = 2 ** 20 - blocksize = 4 * MB # 4MB - size = self.size - progr_size = (size + MB - 1) // MB # in MB - progressbar = self.out.Progress(progr_size, "Dumping image file", 'mb') - - with open(self.real_device, 'r') as src: - with open(outfile, "w") as dst: - left = size - offset = 0 - progressbar.next() - while left > 0: - length = min(left, blocksize) - sent = sendfile(dst.fileno(), src.fileno(), offset, length) - - # Workaround for python-sendfile API change. In - # python-sendfile 1.2.x (py-sendfile) the returning value - # of sendfile is a tuple, where in version 2.x (pysendfile) - # it is just a sigle integer. - if isinstance(sent, tuple): - sent = sent[1] - - offset += sent - left -= sent - progressbar.goto((size - left) // MB) - progressbar.success('image file %s was successfully created' % outfile) + self._images.remove(image) + image.destroy() # vim: set sta sts=4 shiftwidth=4 sw=4 et ai : diff --git a/image_creator/image.py b/image_creator/image.py new file mode 100644 index 0000000..3393efe --- /dev/null +++ b/image_creator/image.py @@ -0,0 +1,372 @@ +# Copyright 2013 GRNET S.A. All rights reserved. +# +# Redistribution and use in source and binary forms, with or +# without modification, are permitted provided that the following +# conditions are met: +# +# 1. Redistributions of source code must retain the above +# copyright notice, this list of conditions and the following +# disclaimer. +# +# 2. Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials +# provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS +# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF +# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED +# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN +# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# +# The views and conclusions contained in the software and +# documentation are those of the authors and should not be +# interpreted as representing official policies, either expressed +# or implied, of GRNET S.A. + +from image_creator.util import FatalError +from image_creator.gpt import GPTPartitionTable +from image_creator.os_type import os_cls + +import re +import guestfs +from sendfile import sendfile + + +class Image(object): + """The instances of this class can create images out of block devices.""" + + def __init__(self, device, output, bootable=True, meta={}): + """Create a new ImageCreator.""" + + self.device = device + self.out = output + self.bootable = bootable + self.meta = meta + self.progress_bar = None + self.guestfs_device = None + self.size = 0 + self.mounted = False + + self.g = guestfs.GuestFS() + self.g.add_drive_opts(self.device, readonly=0, format="raw") + + # Before version 1.17.14 the recovery process, which is a fork of the + # original process that called libguestfs, did not close its inherited + # file descriptors. This can cause problems especially if the parent + # process has opened pipes. Since the recovery process is an optional + # feature of libguestfs, it's better to disable it. + self.g.set_recovery_proc(0) + version = self.g.version() + if version['major'] > 1 or \ + (version['major'] == 1 and (version['minor'] >= 18 or + (version['minor'] == 17 and + version['release'] >= 14))): + self.g.set_recovery_proc(1) + self.out.output("Enabling recovery proc") + + #self.g.set_trace(1) + #self.g.set_verbose(1) + + self.guestfs_enabled = False + + def enable(self): + """Enable a newly created ImageCreator""" + + self.out.output('Launching helper VM (may take a while) ...', False) + # self.progressbar = self.out.Progress(100, "Launching helper VM", + # "percent") + # eh = self.g.set_event_callback(self.progress_callback, + # guestfs.EVENT_PROGRESS) + self.g.launch() + self.guestfs_enabled = True + # self.g.delete_event_callback(eh) + # self.progressbar.success('done') + # self.progressbar = None + self.out.success('done') + + self.out.output('Inspecting Operating System ...', False) + roots = self.g.inspect_os() + if len(roots) == 0: + raise FatalError("No operating system found") + if len(roots) > 1: + raise FatalError("Multiple operating systems found." + "We only support images with one OS.") + self.root = roots[0] + self.guestfs_device = self.g.part_to_dev(self.root) + self.size = self.g.blockdev_getsize64(self.guestfs_device) + self.meta['PARTITION_TABLE'] = \ + self.g.part_get_parttype(self.guestfs_device) + + self.ostype = self.g.inspect_get_type(self.root) + self.distro = self.g.inspect_get_distro(self.root) + self.out.success('found a(n) %s system' % self.distro) + + def _get_os(self): + if hasattr(self, "_os"): + return self._os + + if not self.guestfs_enabled: + self.enable() + + if not self.mounted: + do_unmount = True + self.mount(readonly=True) + else: + do_unmount = False + + try: + cls = os_cls(self.distro, self.ostype) + self._os = cls(self.root, self.g, self.out) + + finally: + if do_unmount: + self.umount() + + return self._os + + os = property(_get_os) + + def destroy(self): + """Destroy this ImageCreator instance.""" + + # In new guestfs versions, there is a handy shutdown method for this + try: + if self.guestfs_enabled: + self.g.umount_all() + self.g.sync() + finally: + # Close the guestfs handler if open + self.g.close() + +# def progress_callback(self, ev, eh, buf, array): +# position = array[2] +# total = array[3] +# +# self.progressbar.goto((position * 100) // total) + + def mount(self, readonly=False): + """Mount all disk partitions in a correct order.""" + + mount = self.g.mount_ro if readonly else self.g.mount + msg = " read-only" if readonly else "" + self.out.output("Mounting the media%s ..." % msg, False) + mps = self.g.inspect_get_mountpoints(self.root) + + # Sort the keys to mount the fs in a correct order. + # / should be mounted befor /boot, etc + def compare(a, b): + if len(a[0]) > len(b[0]): + return 1 + elif len(a[0]) == len(b[0]): + return 0 + else: + return -1 + mps.sort(compare) + for mp, dev in mps: + try: + mount(dev, mp) + except RuntimeError as msg: + self.out.warn("%s (ignored)" % msg) + + self.mounted = True + self.out.success("done") + + def umount(self): + """Umount all mounted filesystems.""" + self.g.umount_all() + self.mounted = False + + def _last_partition(self): + if self.meta['PARTITION_TABLE'] not in 'msdos' 'gpt': + msg = "Unsupported partition table: %s. Only msdos and gpt " \ + "partition tables are supported" % self.meta['PARTITION_TABLE'] + raise FatalError(msg) + + is_extended = lambda p: \ + self.g.part_get_mbr_id(self.guestfs_device, p['part_num']) \ + in (0x5, 0xf) + is_logical = lambda p: \ + self.meta['PARTITION_TABLE'] == 'msdos' and p['part_num'] > 4 + + partitions = self.g.part_list(self.guestfs_device) + last_partition = partitions[-1] + + if is_logical(last_partition): + # The disk contains extended and logical partitions.... + extended = filter(is_extended, partitions)[0] + last_primary = [p for p in partitions if p['part_num'] <= 4][-1] + + # check if extended is the last primary partition + if last_primary['part_num'] > extended['part_num']: + last_partition = last_primary + + return last_partition + + def shrink(self): + """Shrink the disk. + + This is accomplished by shrinking the last filesystem in the + disk and then updating the partition table. The new disk size + (in bytes) is returned. + + ATTENTION: make sure unmount is called before shrink + """ + get_fstype = lambda p: \ + self.g.vfs_type("%s%d" % (self.guestfs_device, p['part_num'])) + is_logical = lambda p: \ + self.meta['PARTITION_TABLE'] == 'msdos' and p['part_num'] > 4 + is_extended = lambda p: \ + self.meta['PARTITION_TABLE'] == 'msdos' and \ + self.g.part_get_mbr_id(self.guestfs_device, p['part_num']) \ + in (0x5, 0xf) + + part_add = lambda ptype, start, stop: \ + self.g.part_add(self.guestfs_device, ptype, start, stop) + part_del = lambda p: self.g.part_del(self.guestfs_device, p) + part_get_id = lambda p: self.g.part_get_mbr_id(self.guestfs_device, p) + part_set_id = lambda p, id: \ + self.g.part_set_mbr_id(self.guestfs_device, p, id) + part_get_bootable = lambda p: \ + self.g.part_get_bootable(self.guestfs_device, p) + part_set_bootable = lambda p, bootable: \ + self.g.part_set_bootable(self.guestfs_device, p, bootable) + + MB = 2 ** 20 + + self.out.output("Shrinking image (this may take a while) ...", False) + + sector_size = self.g.blockdev_getss(self.guestfs_device) + + last_part = None + fstype = None + while True: + last_part = self._last_partition() + fstype = get_fstype(last_part) + + if fstype == 'swap': + self.meta['SWAP'] = "%d:%s" % \ + (last_part['part_num'], + (last_part['part_size'] + MB - 1) // MB) + part_del(last_part['part_num']) + continue + elif is_extended(last_part): + part_del(last_part['part_num']) + continue + + # Most disk manipulation programs leave 2048 sectors after the last + # partition + new_size = last_part['part_end'] + 1 + 2048 * sector_size + self.size = min(self.size, new_size) + break + + if not re.match("ext[234]", fstype): + self.out.warn("Don't know how to resize %s partitions." % fstype) + return self.size + + part_dev = "%s%d" % (self.guestfs_device, last_part['part_num']) + self.g.e2fsck_f(part_dev) + self.g.resize2fs_M(part_dev) + + out = self.g.tune2fs_l(part_dev) + block_size = int(filter(lambda x: x[0] == 'Block size', out)[0][1]) + block_cnt = int(filter(lambda x: x[0] == 'Block count', out)[0][1]) + + start = last_part['part_start'] / sector_size + end = start + (block_size * block_cnt) / sector_size - 1 + + if is_logical(last_part): + partitions = self.g.part_list(self.guestfs_device) + + logical = [] # logical partitions + for partition in partitions: + if partition['part_num'] < 4: + continue + logical.append({ + 'num': partition['part_num'], + 'start': partition['part_start'] / sector_size, + 'end': partition['part_end'] / sector_size, + 'id': part_get_id(partition['part_num']), + 'bootable': part_get_bootable(partition['part_num']) + }) + + logical[-1]['end'] = end # new end after resize + + # Recreate the extended partition + extended = filter(is_extended, partitions)[0] + part_del(extended['part_num']) + part_add('e', extended['part_start'] / sector_size, end) + + # Create all the logical partitions back + for l in logical: + part_add('l', l['start'], l['end']) + part_set_id(l['num'], l['id']) + part_set_bootable(l['num'], l['bootable']) + else: + # Recreate the last partition + if self.meta['PARTITION_TABLE'] == 'msdos': + last_part['id'] = part_get_id(last_part['part_num']) + + last_part['bootable'] = part_get_bootable(last_part['part_num']) + part_del(last_part['part_num']) + part_add('p', start, end) + part_set_bootable(last_part['part_num'], last_part['bootable']) + + if self.meta['PARTITION_TABLE'] == 'msdos': + part_set_id(last_part['part_num'], last_part['id']) + + new_size = (end + 1) * sector_size + + assert (new_size <= self.size) + + if self.meta['PARTITION_TABLE'] == 'gpt': + ptable = GPTPartitionTable(self.device) + self.size = ptable.shrink(new_size, self.size) + else: + self.size = min(new_size + 2048 * sector_size, self.size) + + self.out.success("new size is %dMB" % ((self.size + MB - 1) // MB)) + + return self.size + + def dump(self, outfile): + """Dumps the content of device into a file. + + This method will only dump the actual payload, found by reading the + partition table. Empty space in the end of the device will be ignored. + """ + MB = 2 ** 20 + blocksize = 4 * MB # 4MB + size = self.size + progr_size = (size + MB - 1) // MB # in MB + progressbar = self.out.Progress(progr_size, "Dumping image file", 'mb') + + with open(self.device, 'r') as src: + with open(outfile, "w") as dst: + left = size + offset = 0 + progressbar.next() + while left > 0: + length = min(left, blocksize) + sent = sendfile(dst.fileno(), src.fileno(), offset, length) + + # Workaround for python-sendfile API change. In + # python-sendfile 1.2.x (py-sendfile) the returning value + # of sendfile is a tuple, where in version 2.x (pysendfile) + # it is just a sigle integer. + if isinstance(sent, tuple): + sent = sent[1] + + offset += sent + left -= sent + progressbar.goto((size - left) // MB) + progressbar.success('image file %s was successfully created' % outfile) + +# vim: set sta sts=4 shiftwidth=4 sw=4 et ai : diff --git a/image_creator/main.py b/image_creator/main.py index 497fedf..4b86f7c 100644 --- a/image_creator/main.py +++ b/image_creator/main.py @@ -38,7 +38,6 @@ from image_creator.disk import Disk from image_creator.util import FatalError, MD5 from image_creator.output.cli import SilentOutput, SimpleOutput, \ OutputWthProgress -from image_creator.os_type import os_cls from image_creator.kamaki_wrapper import Kamaki, ClientError import sys import os @@ -205,51 +204,47 @@ def image_creator(): try: snapshot = disk.snapshot() - dev = disk.get_device(snapshot) + image = disk.get_image(snapshot) # If no customization is to be applied, the image should be mounted ro - readonly = (not (options.sysprep or options.shrink) or - options.print_sysprep) - dev.mount(readonly) - - cls = os_cls(dev.distro, dev.ostype) - image_os = cls(dev.root, dev.g, out) - out.output() - - for sysprep in options.disabled_syspreps: - image_os.disable_sysprep(image_os.get_sysprep_by_name(sysprep)) + ro = (not (options.sysprep or options.shrink) or options.print_sysprep) + image.mount(ro) + try: + for sysprep in options.disabled_syspreps: + image.os.disable_sysprep(image.os.get_sysprep_by_name(sysprep)) - for sysprep in options.enabled_syspreps: - image_os.enable_sysprep(image_os.get_sysprep_by_name(sysprep)) + for sysprep in options.enabled_syspreps: + image.os.enable_sysprep(image.os.get_sysprep_by_name(sysprep)) - if options.print_sysprep: - image_os.print_syspreps() - out.output() + if options.print_sysprep: + image.os.print_syspreps() + out.output() - if options.outfile is None and not options.upload: - return 0 + if options.outfile is None and not options.upload: + return 0 - if options.sysprep: - image_os.do_sysprep() + if options.sysprep: + image.os.do_sysprep() - metadata = image_os.meta - dev.umount() + metadata = image.os.meta + finally: + image.umount() - size = options.shrink and dev.shrink() or dev.size - metadata.update(dev.meta) + size = options.shrink and image.shrink() or image.size + metadata.update(image.meta) # Add command line metadata to the collected ones... metadata.update(options.metadata) md5 = MD5(out) - checksum = md5.compute(snapshot, size) + checksum = md5.compute(image.device, size) metastring = '\n'.join( ['%s=%s' % (key, value) for (key, value) in metadata.items()]) metastring += '\n' if options.outfile is not None: - dev.dump(options.outfile) + image.dump(options.outfile) out.output('Dumping metadata file ...', False) with open('%s.%s' % (options.outfile, 'meta'), 'w') as f: @@ -262,8 +257,8 @@ def image_creator(): os.path.basename(options.outfile))) out.success('done') - # Destroy the device. We only need the snapshot from now on - disk.destroy_device(dev) + # Destroy the image instance. We only need the snapshot from now on + disk.destroy_image(image) out.output() try: -- 1.7.10.4