# interpreted as representing official policies, either expressed
# or implied, of GRNET S.A.
-from image_creator.util import get_command, warn, progress_generator
-from image_creator import FatalError
-from clint.textui import indent, puts, colored
-
+from image_creator.util import get_command
+from image_creator.util import FatalError
+from image_creator.gpt import GPTPartitionTable
import stat
import os
import tempfile
import sys
import guestfs
import time
+from sendfile import sendfile
class DiskError(Exception):
the Linux kernel.
"""
- def __init__(self, source):
+ def __init__(self, source, output):
"""Create a new Disk instance out of a source media. The source
media can be an image file, a block device or a directory."""
self._cleanup_jobs = []
self._devices = []
self.source = source
+ self.out = output
def _add_cleanup(self, job, *args):
self._cleanup_jobs.append((job, args))
job, args = self._cleanup_jobs.pop()
job(*args)
- def get_device(self):
- """Returns a newly created DiskDevice instance.
-
- This instance is a snapshot of the original source media of
- the Disk instance.
+ def snapshot(self):
+ """Creates a snapshot of the original source media of the Disk
+ instance.
"""
- puts("Examining source media `%s'" % self.source)
- with indent(4):
- sourcedev = self.source
- mode = os.stat(self.source).st_mode
- if stat.S_ISDIR(mode):
- puts(colored.green('Looks like a directory'))
- return self._losetup(self._dir_to_disk())
- elif stat.S_ISREG(mode):
- puts(colored.green('Looks like an image file'))
- sourcedev = self._losetup(self.source)
- elif not stat.S_ISBLK(mode):
- raise ValueError("Invalid media source. Only block devices, "
- "regular files and directories are supported.")
- else:
- puts(colored.green('Looks like a block device'))
- #puts()
+ self.out.output("Examining source media `%s'..." % self.source, False)
+ sourcedev = self.source
+ mode = os.stat(self.source).st_mode
+ if stat.S_ISDIR(mode):
+ success('looks like a directory')
+ return self._losetup(self._dir_to_disk())
+ elif stat.S_ISREG(mode):
+ success('looks like an image file')
+ sourcedev = self._losetup(self.source)
+ elif not stat.S_ISBLK(mode):
+ raise ValueError("Invalid media source. Only block devices, "
+ "regular files and directories are supported.")
+ else:
+ self.out.success('looks like a block device')
# Take a snapshot and return it to the user
- puts("Snapshotting media source")
- with indent(4):
- size = blockdev('--getsize', sourcedev)
- cowfd, cow = tempfile.mkstemp()
- os.close(cowfd)
- self._add_cleanup(os.unlink, cow)
- # Create 1G cow sparse file
- dd('if=/dev/null', 'of=%s' % cow, 'bs=1k', \
- 'seek=%d' % (1024 * 1024))
- cowdev = self._losetup(cow)
-
- snapshot = uuid.uuid4().hex
- tablefd, table = tempfile.mkstemp()
- try:
- os.write(tablefd, "0 %d snapshot %s %s n 8" % \
- (int(size), sourcedev, cowdev))
- dmsetup('create', snapshot, table)
- self._add_cleanup(dmsetup, 'remove', snapshot)
- # Sometimes dmsetup remove fails with Device or resource busy,
- # although everything is cleaned up and the snapshot is not
- # used by anyone. Add a 2 seconds delay to be on the safe side.
- self._add_cleanup(time.sleep, 2)
-
- finally:
- os.unlink(table)
- puts(colored.green('Done'))
- # puts()
- new_device = DiskDevice("/dev/mapper/%s" % snapshot)
+ self.out.output("Snapshotting media source...", False)
+ size = blockdev('--getsize', sourcedev)
+ cowfd, cow = tempfile.mkstemp()
+ os.close(cowfd)
+ self._add_cleanup(os.unlink, cow)
+ # Create 1G cow sparse file
+ dd('if=/dev/null', 'of=%s' % cow, 'bs=1k', 'seek=%d' % (1024 * 1024))
+ cowdev = self._losetup(cow)
+
+ snapshot = uuid.uuid4().hex
+ tablefd, table = tempfile.mkstemp()
+ try:
+ os.write(tablefd, "0 %d snapshot %s %s n 8" % \
+ (int(size), sourcedev, cowdev))
+ dmsetup('create', snapshot, table)
+ self._add_cleanup(dmsetup, 'remove', snapshot)
+ # Sometimes dmsetup remove fails with Device or resource busy,
+ # although everything is cleaned up and the snapshot is not
+ # used by anyone. Add a 2 seconds delay to be on the safe side.
+ self._add_cleanup(time.sleep, 2)
+
+ finally:
+ os.unlink(table)
+ self.out.success('done')
+ return "/dev/mapper/%s" % snapshot
+
+ def get_device(self, media):
+ """Returns a newly created DiskDevice instance."""
+
+ new_device = DiskDevice(media, self.out)
self._devices.append(new_device)
new_device.enable()
return new_device
as created by the device-mapper.
"""
- def __init__(self, device, bootable=True):
+ def __init__(self, device, output, bootable=True):
"""Create a new DiskDevice."""
- self.device = device
+ self.real_device = device
+ self.out = output
self.bootable = bootable
self.progress_bar = None
+ self.guestfs_device = None
+ self.meta = {}
self.g = guestfs.GuestFS()
- self.g.add_drive_opts(self.device, readonly=0)
+ self.g.add_drive_opts(self.real_device, readonly=0)
+
+ # Before version 1.17.14 the recovery process, which is a fork of the
+ # original process that called libguestfs, did not close its inherited
+ # file descriptors. This can cause problems especially if the parent
+ # process has opened pipes. Since the recovery process is an optional
+ # feature of libguestfs, it's better to disable it.
+ self.g.set_recovery_proc(0)
+ version = self.g.version()
+ if version['major'] > 1 or (version['major'] == 1 and
+ (version['minor'] >= 18 or \
+ (version['minor'] == 17 and version['release'] >= 14))):
+ self.g.set_recovery_proc(1)
+ self.out.output("Enabling recovery proc")
#self.g.set_trace(1)
#self.g.set_verbose(1)
def enable(self):
"""Enable a newly created DiskDevice"""
- self.progressbar = progress_generator("Launching helper VM: ")
- with indent(4):
- self.progressbar.next()
- eh = self.g.set_event_callback(self.progress_callback,
- guestfs.EVENT_PROGRESS)
- self.g.launch()
- self.guestfs_enabled = True
- self.g.delete_event_callback(eh)
- if self.progressbar is not None:
- self.progressbar.send(100)
- self.progressbar = None
- puts(colored.green('Done'))
-
- puts('Inspecting Operating System')
- with indent(4):
- roots = self.g.inspect_os()
- if len(roots) == 0:
- raise FatalError("No operating system found")
- if len(roots) > 1:
- raise FatalError("Multiple operating systems found."
- "We only support images with one filesystem.")
- self.root = roots[0]
- self.ostype = self.g.inspect_get_type(self.root)
- self.distro = self.g.inspect_get_distro(self.root)
- puts(colored.green('Found a %s system' % self.distro))
- puts()
+ self.progressbar = self.out.Progress(100, "Launching helper VM",
+ "percent")
+ eh = self.g.set_event_callback(self.progress_callback,
+ guestfs.EVENT_PROGRESS)
+ self.g.launch()
+ self.guestfs_enabled = True
+ self.g.delete_event_callback(eh)
+ self.progressbar.success('done')
+ self.progressbar = None
+
+ self.out.output('Inspecting Operating System...', False)
+ roots = self.g.inspect_os()
+ if len(roots) == 0:
+ raise FatalError("No operating system found")
+ if len(roots) > 1:
+ raise FatalError("Multiple operating systems found."
+ "We only support images with one filesystem.")
+ self.root = roots[0]
+ self.guestfs_device = self.g.part_to_dev(self.root)
+ self.meta['SIZE'] = self.g.blockdev_getsize64(self.guestfs_device)
+ self.meta['PARTITION_TABLE'] = \
+ self.g.part_get_parttype(self.guestfs_device)
+
+ self.ostype = self.g.inspect_get_type(self.root)
+ self.distro = self.g.inspect_get_distro(self.root)
+ self.out.success('found a(n) %s system' % self.distro)
def destroy(self):
"""Destroy this DiskDevice instance."""
position = array[2]
total = array[3]
- self.progressbar.send((position * 100) // total)
-
- if position == total:
- self.progressbar = None
+ self.progressbar.goto((position * 100) // total)
- def mount(self):
+ def mount(self, readonly=False):
"""Mount all disk partitions in a correct order."""
+
+ mount = self.g.mount_ro if readonly else self.g.mount
+ self.out.output("Mounting image...", False)
mps = self.g.inspect_get_mountpoints(self.root)
# Sort the keys to mount the fs in a correct order.
mps.sort(compare)
for mp, dev in mps:
try:
- self.g.mount(dev, mp)
+ mount(dev, mp)
except RuntimeError as msg:
- print "%s (ignored)" % msg
+ self.out.warn("%s (ignored)" % msg)
+ self.out.success("done")
def umount(self):
"""Umount all mounted filesystems."""
self.g.umount_all()
- def shrink(self):
- """Shrink the disk.
+ def _last_partition(self):
+ if self.meta['PARTITION_TABLE'] not in 'msdos' 'gpt':
+ msg = "Unsupported partition table: %s. Only msdos and gpt " \
+ "partition tables are supported" % self.meta['PARTITION_TABLE']
+ raise FatalError(msg)
- This is accomplished by shrinking the last filesystem in the
- disk and then updating the partition table. The new disk size
- (in bytes) is returned.
- """
- puts("Shrinking image (this may take a while)")
-
- dev = self.g.part_to_dev(self.root)
- parttype = self.g.part_get_parttype(dev)
- if parttype != 'msdos':
- raise FatalError("You have a %s partition table. "
- "Only msdos partitions are supported" % parttype)
+ is_extended = lambda p: self.g.part_get_mbr_id(
+ self.guestfs_device, p['part_num']) == 5
+ is_logical = lambda p: self.meta['PARTITION_TABLE'] != 'msdos' and \
+ p['part_num'] > 4
- last_partition = self.g.part_list(dev)[-1]
+ partitions = self.g.part_list(self.guestfs_device)
+ last_partition = partitions[-1]
- if last_partition['part_num'] > 4:
- raise FatalError("This disk contains logical partitions. "
- "Only primary partitions are supported.")
+ if is_logical(last_partition):
+ # The disk contains extended and logical partitions....
+ extended = [p for p in partitions if is_extended(p)][0]
+ last_primary = [p for p in partitions if p['part_num'] <= 4][-1]
- part_dev = "%s%d" % (dev, last_partition['part_num'])
- fs_type = self.g.vfs_type(part_dev)
- if not re.match("ext[234]", fs_type):
- warn("Don't know how to resize %s partitions." % vfs_type)
- return
+ # check if extended is the last primary partition
+ if last_primary['part_num'] > extended['part_num']:
+ last_partition = last_primary
- with indent(4):
- self.g.e2fsck_f(part_dev)
- self.g.resize2fs_M(part_dev)
+ return last_partition
- output = self.g.tune2fs_l(part_dev)
- block_size = int(
- filter(lambda x: x[0] == 'Block size', output)[0][1])
- block_cnt = int(
- filter(lambda x: x[0] == 'Block count', output)[0][1])
-
- sector_size = self.g.blockdev_getss(dev)
-
- start = last_partition['part_start'] / sector_size
- end = start + (block_size * block_cnt) / sector_size - 1
-
- self.g.part_del(dev, last_partition['part_num'])
- self.g.part_add(dev, 'p', start, end)
-
- new_size = (end + 1) * sector_size
- puts(" New image size is %dMB\n" % (new_size // 2 ** 20))
- return new_size
+ def shrink(self):
+ """Shrink the disk.
- def size(self):
- """Returns the "payload" size of the device.
+ This is accomplished by shrinking the last filesystem in the
+ disk and then updating the partition table. The new disk size
+ (in bytes) is returned.
- The size returned by this method is the size of the space occupied by
- the partitions (including the space before the first partition).
+ ATTENTION: make sure unmount is called before shrink
"""
- dev = self.g.part_to_dev(self.root)
- last = self.g.part_list(dev)[-1]
-
- return last['part_end']
+ get_fstype = lambda p: self.g.vfs_type("%s%d" % \
+ (self.guestfs_device, p['part_num']))
+ is_logical = lambda p: self.meta['PARTITION_TABLE'] == 'msdos' and \
+ p['part_num'] > 4
+ is_extended = lambda p: self.meta['PARTITION_TABLE'] == 'msdos' and \
+ self.g.part_get_mbr_id(self.guestfs_device, p['part_num']) == 5
+
+ part_add = lambda ptype, start, stop: \
+ self.g.part_add(self.guestfs_device, ptype, start, stop)
+ part_del = lambda p: self.g.part_del(self.guestfs_device, p)
+ part_get_id = lambda p: self.g.part_get_mbr_id(self.guestfs_device, p)
+ part_set_id = lambda p, id: self.g.part_set_mbr_id(
+ self.guestfs_device, p, id)
+ part_get_bootable = lambda p: self.g.part_get_bootable(
+ self.guestfs_device, p)
+ part_set_bootable = lambda p, bootable: self.g.part_set_bootable(
+ self.guestfs_device, p, bootable)
+
+ MB = 2 ** 20
+
+ self.out.output("Shrinking image (this may take a while)...", False)
+
+ last_part = None
+ fstype = None
+ while True:
+ last_part = self._last_partition()
+ fstype = get_fstype(last_part)
+
+ if fstype == 'swap':
+ self.meta['SWAP'] = "%d:%s" % \
+ (last_part['part_num'],
+ (last_part['part_size'] + MB - 1) // MB)
+ part_del(last_part['part_num'])
+ continue
+ elif is_extended(last_part):
+ part_del(last_part['part_num'])
+ continue
+
+ self.meta['SIZE'] = last_part['part_end'] + 1
+ break
+
+ if not re.match("ext[234]", fstype):
+ self.out.warn("Don't know how to resize %s partitions." % fstype)
+ return self.meta['SIZE']
+
+ part_dev = "%s%d" % (self.guestfs_device, last_part['part_num'])
+ self.g.e2fsck_f(part_dev)
+ self.g.resize2fs_M(part_dev)
+
+ out = self.g.tune2fs_l(part_dev)
+ block_size = int(
+ filter(lambda x: x[0] == 'Block size', out)[0][1])
+ block_cnt = int(
+ filter(lambda x: x[0] == 'Block count', out)[0][1])
+
+ sector_size = self.g.blockdev_getss(self.guestfs_device)
+ start = last_part['part_start'] / sector_size
+ end = start + (block_size * block_cnt) / sector_size - 1
+
+ if is_logical(last_part):
+ partitions = self.g.part_list(self.guestfs_device)
+
+ logical = [] # logical partitions
+ for partition in partitions:
+ if partition['part_num'] < 4:
+ continue
+ logical.append({
+ 'num': partition['part_num'],
+ 'start': partition['part_start'] / sector_size,
+ 'end': partition['part_end'] / sector_size,
+ 'id': part_get_(partition['part_num']),
+ 'bootable': part_get_bootable(partition['part_num'])
+ })
+
+ logical[-1]['end'] = end # new end after resize
+
+ # Recreate the extended partition
+ extended = [p for p in partitions if self._is_extended(p)][0]
+ part_del(extended['part_num'])
+ part_add('e', extended['part_start'], end)
+
+ # Create all the logical partitions back
+ for l in logical:
+ part_add('l', l['start'], l['end'])
+ part_set_id(l['num'], l['id'])
+ part_set_bootable(l['num'], l['bootable'])
+ else:
+ # Recreate the last partition
+ if self.meta['PARTITION_TABLE'] == 'msdos':
+ last_part['id'] = part_get_id(last_part['part_num'])
+
+ last_part['bootable'] = part_get_bootable(last_part['part_num'])
+ part_del(last_part['part_num'])
+ part_add('p', start, end)
+ part_set_bootable(last_part['part_num'], last_part['bootable'])
+
+ if self.meta['PARTITION_TABLE'] == 'msdos':
+ part_set_id(last_part['part_num'], last_part['id'])
+
+ new_size = (end + 1) * sector_size
+ self.out.success("new size is %dMB" % ((new_size + MB - 1) // MB))
+
+ if self.meta['PARTITION_TABLE'] == 'gpt':
+ ptable = GPTPartitionTable(self.real_device)
+ self.meta['SIZE'] = ptable.shrink(new_size)
+ else:
+ self.meta['SIZE'] = new_size
+
+ return self.meta['SIZE']
+
+ def dump(self, outfile):
+ """Dumps the content of device into a file.
+
+ This method will only dump the actual payload, found by reading the
+ partition table. Empty space in the end of the device will be ignored.
+ """
+ MB = 2 ** 20
+ blocksize = 4 * MB # 4MB
+ size = self.meta['SIZE']
+ progr_size = (size + MB - 1) // MB # in MB
+ progressbar = self.out.Progress(progr_size, "Dumping image file", 'mb')
+
+ with open(self.real_device, 'r') as src:
+ with open(outfile, "w") as dst:
+ left = size
+ offset = 0
+ progressbar.next()
+ while left > 0:
+ length = min(left, blocksize)
+ sent = sendfile(dst.fileno(), src.fileno(), offset, length)
+ offset += sent
+ left -= sent
+ progressbar.goto((size - left) // MB)
+ progressbar.success('image file %s was successfully created' % outfile)
# vim: set sta sts=4 shiftwidth=4 sw=4 et ai :