X-Git-Url: https://code.grnet.gr/git/snf-image-creator/blobdiff_plain/25b4d8581861e089cffd3e78b0823d93787bef7f..99f95b85b1afd94926553ea21b472311bbd3c66b:/image_creator/bundle_volume.py diff --git a/image_creator/bundle_volume.py b/image_creator/bundle_volume.py index ee93fc8..c099daa 100644 --- a/image_creator/bundle_volume.py +++ b/image_creator/bundle_volume.py @@ -33,44 +33,46 @@ import os import re -import uuid import tempfile -import time from collections import namedtuple import parted +from image_creator.rsync import Rsync from image_creator.util import get_command from image_creator.util import FatalError +from image_creator.util import try_fail_repeat +from image_creator.util import free_space findfs = get_command('findfs') -truncate = get_command('truncate') dd = get_command('dd') dmsetup = get_command('dmsetup') losetup = get_command('losetup') mount = get_command('mount') umount = get_command('umount') - -MKFS_OPTS = { - 'ext2': ['-F'], - 'ext3': ['-F'], - 'ext4': ['-F'], - 'reiserfs': ['-ff'], - 'btrfs': [], - 'minix': [], - 'xfs': ['-f'], - 'jfs': ['-f'], - 'ntfs': ['-F'], - 'msdos': [], - 'vfat': [] - } - - -class BundleVolume(): - - def __init__(self, out, meta): +blkid = get_command('blkid') + +MKFS_OPTS = {'ext2': ['-F'], + 'ext3': ['-F'], + 'ext4': ['-F'], + 'reiserfs': ['-ff'], + 'btrfs': [], + 'minix': [], + 'xfs': ['-f'], + 'jfs': ['-f'], + 'ntfs': ['-F'], + 'msdos': [], + 'vfat': []} + + +class BundleVolume(object): + """This class can be used to create an image out of the running system""" + + def __init__(self, out, meta, tmp=None): + """Create an instance of the BundleVolume class.""" self.out = out self.meta = meta + self.tmp = tmp self.out.output('Searching for root device ...', False) root = self._get_root_partition() @@ -92,14 +94,14 @@ class BundleVolume(): if not os.path.isfile(f): raise FatalError("Unable to open: `%s'. File is missing." % f) - FileSystemEntry = namedtuple('FileSystemEntry', - 'dev mpoint fs opts freq passno') + FileSystemTableEntry = namedtuple('FileSystemTableEntry', + 'dev mpoint fs opts freq passno') with open(f) as table: for line in iter(table): entry = line.split('#')[0].strip().split() if len(entry) != 6: continue - yield FileSystemEntry(*entry) + yield FileSystemTableEntry(*entry) def _get_root_partition(self): for entry in self._read_fstable('/etc/fstab'): @@ -201,8 +203,8 @@ class BundleVolume(): new_end = last.end + 2048 mount_options = self._get_mount_options( - self.disk.getPartitionBySector(last.start).path) - if mount_options is not None: + self.disk.getPartitionBySector(last.start).path) + if mount_options is not None: stat = os.statvfs(mount_options.mpoint) # Shrink the last partition. The new size should be the size of the # occupied blocks @@ -211,13 +213,13 @@ class BundleVolume(): # Add 10% just to be on the safe side part_end = last.start + (new_size * 11) // 10 - # Alighn to 2048 + # Align to 2048 part_end = ((part_end + 2047) // 2048) * 2048 image_disk.setPartitionGeometry( image_disk.getPartitionBySector(last.start), parted.Constraint(device=image_disk.device), - start=last.start, end=last.end) + start=last.start, end=part_end) image_disk.commit() # Parted may have changed this for better alignment @@ -226,18 +228,18 @@ class BundleVolume(): partitions[-1] = last # Leave 2048 blocks at the end. - new_end = new_size + 2048 - + new_end = part_end + 2048 if last.type == parted.PARTITION_LOGICAL: # Fix the extended partition extended = disk.getExtendedPartition() - image_disk.setPartitionGeometry(extended, - parted.Constraint(device=img_dev), + image_disk.setPartitionGeometry( + extended, parted.Constraint(device=img_dev), ext.geometry.start, end=last.end) image_disk.commit() + image_dev.destroy() return new_end def _map_partition(self, dev, num, start, end): @@ -256,8 +258,7 @@ class BundleVolume(): if not os.path.exists(dev): return - dmsetup('remove', dev.split('/dev/mapper/')[1]) - time.sleep(0.1) + try_fail_repeat(dmsetup, 'remove', dev.split('/dev/mapper/')[1]) def _mount(self, target, devs): @@ -273,20 +274,81 @@ class BundleVolume(): for entry in self._read_fstable('/proc/mounts'): if entry.mpoint.startswith(os.path.abspath(target)): mpoints.append(entry.mpoint) - + mpoints.sort() for mpoint in reversed(mpoints): - umount(mpoint) + try_fail_repeat(umount, mpoint) + + def _to_exclude(self): + excluded = ['/tmp', '/var/tmp'] + if self.tmp is not None: + excluded.append(self.tmp) + local_filesystems = MKFS_OPTS.keys() + ['rootfs'] + for entry in self._read_fstable('/proc/mounts'): + if entry.fs in local_filesystems: + continue + + mpoint = entry.mpoint + if mpoint in excluded: + continue + + descendants = filter( + lambda p: p.startswith(mpoint + '/'), excluded) + if len(descendants): + for d in descendants: + excluded.remove(d) + excluded.append(mpoint) + continue + + dirname = mpoint + basename = '' + found_ancestor = False + while dirname != '/': + (dirname, basename) = os.path.split(dirname) + if dirname in excluded: + found_ancestor = True + break + + if not found_ancestor: + excluded.append(mpoint) + + return map(lambda d: d + "/*", excluded) + + def _replace_uuids(self, target, new_uuid): + + files = ['/etc/fstab', + '/boot/grub/grub.cfg', + '/boot/grub/menu.lst', + '/boot/grub/grub.conf'] + + orig = dict(map( + lambda p: ( + p.number, + blkid('-s', 'UUID', '-o', 'value', p.path).stdout.strip()), + self.disk.partitions)) + + for f in map(lambda f: target + f, files): + + if not os.path.exists(f): + continue + + with open(f, 'r') as src: + lines = src.readlines() + with open(f, 'w') as dest: + for line in lines: + for i, uuid in new_uuid.items(): + line = re.sub(orig[i], uuid, line) + dest.write(line) def _create_filesystems(self, image): - - partitions = self._get_partitions(parted.Disk(parted.Device(image))) - filesystems = {} + + filesystem = {} for p in self.disk.partitions: - filesystems[p.number] = self._get_mount_options(p.path) + filesystem[p.number] = self._get_mount_options(p.path) - unmounted = filter(lambda p: filesystems[p.num] is None, partitions) - mounted = filter(lambda p: filesystems[p.num] is not None, partitions) + partitions = self._get_partitions(parted.Disk(parted.Device(image))) + unmounted = filter(lambda p: filesystem[p.num] is None, partitions) + mounted = filter(lambda p: filesystem[p.num] is not None, partitions) # For partitions that are not mounted right now, we can simply dd them # into the image. @@ -299,22 +361,40 @@ class BundleVolume(): mapped = {} try: for p in mounted: - i = p.num + i = p.num mapped[i] = self._map_partition(loop, i, p.start, p.end) + new_uuid = {} # Create the file systems for i, dev in mapped.iteritems(): - fs = filesystems[i].fs + fs = filesystem[i].fs self.out.output('Creating %s filesystem on partition %d ... ' % - (fs, i), False) + (fs, i), False) get_command('mkfs.%s' % fs)(*(MKFS_OPTS[fs] + [dev])) self.out.success('done') + new_uuid[i] = blkid( + '-s', 'UUID', '-o', 'value', dev).stdout.strip() target = tempfile.mkdtemp() try: absmpoints = self._mount(target, - [(mapped[i], filesystems[i].mpoint) for i in mapped.keys()] - ) + [(mapped[i], filesystem[i].mpoint) + for i in mapped.keys()]) + exclude = self._to_exclude() + [image] + + rsync = Rsync(self.out) + + # Excluded paths need to be relative to the source + for excl in map(lambda p: os.path.relpath(p, '/'), exclude): + rsync.exclude(excl) + + rsync.archive().hard_links().xattrs().sparse().acls() + rsync.run('/', target, 'host', 'temporary image') + + # We need to replace the old UUID referencies with the new + # ones in grub configuration files and /etc/fstab for file + # systems that have been recreated. + self._replace_uuids(target, new_uuid) finally: self._umount_all(target) @@ -324,26 +404,38 @@ class BundleVolume(): self._unmap_partition(dev) losetup('-d', loop) - def create_image(self): + def create_image(self, image): + """Given an image filename, this method will create an image out of the + running system. + """ - image = '/mnt/%s.diskdump' % uuid.uuid4().hex - - disk_size = self.disk.device.getLength() * self.disk.device.sectorSize + size = self.disk.device.getLength() * self.disk.device.sectorSize # Create sparse file to host the image - truncate("-s", "%d" % disk_size, image) + fd = os.open(image, os.O_WRONLY | os.O_CREAT) + try: + os.ftruncate(fd, size) + finally: + os.close(fd) self._create_partition_table(image) + end_sector = self._shrink_partitions(image) + size = (end_sector + 1) * self.disk.device.sectorSize + + # Truncate image to the new size. + fd = os.open(image, os.O_RDWR) + try: + os.ftruncate(fd, size) + finally: + os.close(fd) + # Check if the available space is enough to host the image dirname = os.path.dirname(image) - size = (end_sector + 1) * self.disk.device.sectorSize - self.out.output("Examining available space in %s ..." % dirname, False) - stat = os.statvfs(dirname) - available = stat.f_bavail * stat.f_frsize - if available <= size: - raise FatalError('Not enough space in %s to host the image' % + self.out.output("Examining available space ...", False) + if free_space(dirname) <= size: + raise FatalError('Not enough space under %s to host the image' % dirname) self.out.success("sufficient")