Bump version to 0.2.2
[snf-image-creator] / image_creator / bundle_volume.py
index fde1e00..4b15acd 100644 (file)
@@ -33,9 +33,7 @@
 
 import os
 import re
-import uuid
 import tempfile
-import time
 from collections import namedtuple
 
 import parted
@@ -43,35 +41,38 @@ import parted
 from image_creator.rsync import Rsync
 from image_creator.util import get_command
 from image_creator.util import FatalError
+from image_creator.util import try_fail_repeat
+from image_creator.util import free_space
 
 findfs = get_command('findfs')
-truncate = get_command('truncate')
 dd = get_command('dd')
 dmsetup = get_command('dmsetup')
 losetup = get_command('losetup')
 mount = get_command('mount')
 umount = get_command('umount')
-
-MKFS_OPTS = {
-    'ext2': ['-F'],
-    'ext3': ['-F'],
-    'ext4': ['-F'],
-    'reiserfs': ['-ff'],
-    'btrfs': [],
-    'minix': [],
-    'xfs': ['-f'],
-    'jfs': ['-f'],
-    'ntfs': ['-F'],
-    'msdos': [],
-    'vfat': []
-    }
-
-
-class BundleVolume():
-
-    def __init__(self, out, meta):
+blkid = get_command('blkid')
+
+MKFS_OPTS = {'ext2': ['-F'],
+             'ext3': ['-F'],
+             'ext4': ['-F'],
+             'reiserfs': ['-ff'],
+             'btrfs': [],
+             'minix': [],
+             'xfs': ['-f'],
+             'jfs': ['-f'],
+             'ntfs': ['-F'],
+             'msdos': [],
+             'vfat': []}
+
+
+class BundleVolume(object):
+    """This class can be used to create an image out of the running system"""
+
+    def __init__(self, out, meta, tmp=None):
+        """Create an instance of the BundleVolume class."""
         self.out = out
         self.meta = meta
+        self.tmp = tmp
 
         self.out.output('Searching for root device ...', False)
         root = self._get_root_partition()
@@ -94,7 +95,7 @@ class BundleVolume():
             raise FatalError("Unable to open: `%s'. File is missing." % f)
 
         FileSystemTableEntry = namedtuple('FileSystemTableEntry',
-                                     'dev mpoint fs opts freq passno')
+                                          'dev mpoint fs opts freq passno')
         with open(f) as table:
             for line in iter(table):
                 entry = line.split('#')[0].strip().split()
@@ -202,7 +203,7 @@ class BundleVolume():
             new_end = last.end + 2048
 
         mount_options = self._get_mount_options(
-                self.disk.getPartitionBySector(last.start).path)
+            self.disk.getPartitionBySector(last.start).path)
         if mount_options is not None:
             stat = os.statvfs(mount_options.mpoint)
             # Shrink the last partition. The new size should be the size of the
@@ -212,13 +213,13 @@ class BundleVolume():
 
             # Add 10% just to be on the safe side
             part_end = last.start + (new_size * 11) // 10
-            # Alighn to 2048
+            # Align to 2048
             part_end = ((part_end + 2047) // 2048) * 2048
 
             image_disk.setPartitionGeometry(
                 image_disk.getPartitionBySector(last.start),
                 parted.Constraint(device=image_disk.device),
-                start=last.start, end=last.end)
+                start=last.start, end=part_end)
             image_disk.commit()
 
             # Parted may have changed this for better alignment
@@ -227,17 +228,18 @@ class BundleVolume():
             partitions[-1] = last
 
             # Leave 2048 blocks at the end.
-            new_end = new_size + 2048
+            new_end = part_end + 2048
 
             if last.type == parted.PARTITION_LOGICAL:
                 # Fix the extended partition
                 extended = disk.getExtendedPartition()
 
-                image_disk.setPartitionGeometry(extended,
-                    parted.Constraint(device=img_dev),
+                image_disk.setPartitionGeometry(
+                    extended, parted.Constraint(device=img_dev),
                     ext.geometry.start, end=last.end)
                 image_disk.commit()
 
+        image_dev.destroy()
         return new_end
 
     def _map_partition(self, dev, num, start, end):
@@ -256,8 +258,7 @@ class BundleVolume():
         if not os.path.exists(dev):
             return
 
-        dmsetup('remove', dev.split('/dev/mapper/')[1])
-        time.sleep(0.1)
+        try_fail_repeat(dmsetup, 'remove', dev.split('/dev/mapper/')[1])
 
     def _mount(self, target, devs):
 
@@ -276,10 +277,12 @@ class BundleVolume():
 
         mpoints.sort()
         for mpoint in reversed(mpoints):
-            umount(mpoint)
+            try_fail_repeat(umount, mpoint)
 
     def _to_exclude(self):
-        excluded = ['/tmp']
+        excluded = ['/tmp', '/var/tmp']
+        if self.tmp is not None:
+            excluded.append(self.tmp)
         local_filesystems = MKFS_OPTS.keys() + ['rootfs']
         for entry in self._read_fstable('/proc/mounts'):
             if entry.fs in local_filesystems:
@@ -289,8 +292,8 @@ class BundleVolume():
             if mpoint in excluded:
                 continue
 
-            descendants = filter(lambda p: p.startswith(mpoint + '/'),
-                    excluded)
+            descendants = filter(
+                lambda p: p.startswith(mpoint + '/'), excluded)
             if len(descendants):
                 for d in descendants:
                     excluded.remove(d)
@@ -309,17 +312,43 @@ class BundleVolume():
             if not found_ancestor:
                 excluded.append(mpoint)
 
-        return map(lambda d: d + "/*", excluded)
+        return excluded
+
+    def _replace_uuids(self, target, new_uuid):
+
+        files = ['/etc/fstab',
+                 '/boot/grub/grub.cfg',
+                 '/boot/grub/menu.lst',
+                 '/boot/grub/grub.conf']
+
+        orig = dict(map(
+            lambda p: (
+                p.number,
+                blkid('-s', 'UUID', '-o', 'value', p.path).stdout.strip()),
+            self.disk.partitions))
+
+        for f in map(lambda f: target + f, files):
+
+            if not os.path.exists(f):
+                continue
+
+            with open(f, 'r') as src:
+                lines = src.readlines()
+            with open(f, 'w') as dest:
+                for line in lines:
+                    for i, uuid in new_uuid.items():
+                        line = re.sub(orig[i], uuid, line)
+                    dest.write(line)
 
     def _create_filesystems(self, image):
 
-        partitions = self._get_partitions(parted.Disk(parted.Device(image)))
-        filesystems = {}
+        filesystem = {}
         for p in self.disk.partitions:
-            filesystems[p.number] = self._get_mount_options(p.path)
+            filesystem[p.number] = self._get_mount_options(p.path)
 
-        unmounted = filter(lambda p: filesystems[p.num] is None, partitions)
-        mounted = filter(lambda p: filesystems[p.num] is not None, partitions)
+        partitions = self._get_partitions(parted.Disk(parted.Device(image)))
+        unmounted = filter(lambda p: filesystem[p.num] is None, partitions)
+        mounted = filter(lambda p: filesystem[p.num] is not None, partitions)
 
         # For partitions that are not mounted right now, we can simply dd them
         # into the image.
@@ -335,23 +364,48 @@ class BundleVolume():
                 i = p.num
                 mapped[i] = self._map_partition(loop, i, p.start, p.end)
 
+            new_uuid = {}
             # Create the file systems
             for i, dev in mapped.iteritems():
-                fs = filesystems[i].fs
+                fs = filesystem[i].fs
                 self.out.output('Creating %s filesystem on partition %d ... ' %
-                    (fs, i), False)
+                                (fs, i), False)
                 get_command('mkfs.%s' % fs)(*(MKFS_OPTS[fs] + [dev]))
                 self.out.success('done')
+                new_uuid[i] = blkid(
+                    '-s', 'UUID', '-o', 'value', dev).stdout.strip()
 
             target = tempfile.mkdtemp()
             try:
                 absmpoints = self._mount(target,
-                    [(mapped[i], filesystems[i].mpoint) for i in mapped.keys()]
-                )
-                exclude = self._to_exclude() + [image]
-                rsync = Rsync('/', target, exclude)
-                msg = "Copying host files into the image"
-                rsync.archive().run(self.out, msg)
+                                         [(mapped[i], filesystem[i].mpoint)
+                                         for i in mapped.keys()])
+                excluded = self._to_exclude()
+
+                rsync = Rsync(self.out)
+
+                # Excluded paths need to be relative to the source
+                for excl in map(lambda p: os.path.relpath(p, '/'),
+                                excluded + [image]):
+                    rsync.exclude(excl)
+
+                rsync.archive().hard_links().xattrs().sparse().acls()
+                rsync.run('/', target, 'host', 'temporary image')
+
+                # Create missing mountpoints. Since they are mountpoints, we
+                # cannot determine the ownership and the mode of the real
+                # directory. Make them inherit those properties from their
+                # parent dir
+                for excl in excluded:
+                   dirname = os.path.dirname(excl)
+                   stat = os.stat(dirname)
+                   os.mkdir(target + excl, stat.st_mode)
+                   os.chown(target + excl, stat.st_uid, stat.st_gid)
+
+                # We need to replace the old UUID referencies with the new
+                # ones in grub configuration files and /etc/fstab for file
+                # systems that have been recreated.
+                self._replace_uuids(target, new_uuid)
 
             finally:
                 self._umount_all(target)
@@ -361,26 +415,38 @@ class BundleVolume():
                 self._unmap_partition(dev)
             losetup('-d', loop)
 
-    def create_image(self):
-
-        image = '/mnt/%s.diskdump' % uuid.uuid4().hex
+    def create_image(self, image):
+        """Given an image filename, this method will create an image out of the
+        running system.
+        """
 
-        disk_size = self.disk.device.getLength() * self.disk.device.sectorSize
+        size = self.disk.device.getLength() * self.disk.device.sectorSize
 
         # Create sparse file to host the image
-        truncate("-s", "%d" % disk_size, image)
+        fd = os.open(image, os.O_WRONLY | os.O_CREAT)
+        try:
+            os.ftruncate(fd, size)
+        finally:
+            os.close(fd)
 
         self._create_partition_table(image)
+
         end_sector = self._shrink_partitions(image)
 
+        size = (end_sector + 1) * self.disk.device.sectorSize
+
+        # Truncate image to the new size.
+        fd = os.open(image, os.O_RDWR)
+        try:
+            os.ftruncate(fd, size)
+        finally:
+            os.close(fd)
+
         # Check if the available space is enough to host the image
         dirname = os.path.dirname(image)
-        size = (end_sector + 1) * self.disk.device.sectorSize
-        self.out.output("Examining available space in %s ..." % dirname, False)
-        stat = os.statvfs(dirname)
-        available = stat.f_bavail * stat.f_frsize
-        if available <= size:
-            raise FatalError('Not enough space in %s to host the image' %
+        self.out.output("Examining available space ...", False)
+        if free_space(dirname) <= size:
+            raise FatalError('Not enough space under %s to host the image' %
                              dirname)
         self.out.success("sufficient")