From: Nikos Skalkotos Date: Thu, 20 Dec 2012 18:16:40 +0000 (+0200) Subject: Create fs in image partitions in bundle_volume X-Git-Tag: v0.2~30 X-Git-Url: https://code.grnet.gr/git/snf-image-creator/commitdiff_plain/25b4d8581861e089cffd3e78b0823d93787bef7f?hp=a939e3b86c906d078be60d8addf60865765804c0 Create fs in image partitions in bundle_volume Also do a code cleanup --- diff --git a/image_creator/bundle_volume.py b/image_creator/bundle_volume.py index 15f16ad..ee93fc8 100644 --- a/image_creator/bundle_volume.py +++ b/image_creator/bundle_volume.py @@ -35,6 +35,7 @@ import os import re import uuid import tempfile +import time from collections import namedtuple import parted @@ -46,13 +47,26 @@ findfs = get_command('findfs') truncate = get_command('truncate') dd = get_command('dd') dmsetup = get_command('dmsetup') +losetup = get_command('losetup') +mount = get_command('mount') +umount = get_command('umount') + +MKFS_OPTS = { + 'ext2': ['-F'], + 'ext3': ['-F'], + 'ext4': ['-F'], + 'reiserfs': ['-ff'], + 'btrfs': [], + 'minix': [], + 'xfs': ['-f'], + 'jfs': ['-f'], + 'ntfs': ['-F'], + 'msdos': [], + 'vfat': [] + } class BundleVolume(): - _FileSystemEntry = namedtuple('FileSystemEntry', - 'dev mpoint fs opts freq passno') - - _Partition = namedtuple('Partition', 'num start end type fs mopts') def __init__(self, out, meta): self.out = out @@ -62,28 +76,30 @@ class BundleVolume(): root = self._get_root_partition() if root.startswith("UUID=") or root.startswith("LABEL="): - self.root = findfs(root).stdout.strip() - else: - self.root = root + root = findfs(root).stdout.strip() - if not re.match('/dev/[hsv]d[a-z][1-9]*$', self.root): - raise FatalError("Don't know how to handle root device: %s" % \ - self.root) + if not re.match('/dev/[hsv]d[a-z][1-9]*$', root): + raise FatalError("Don't know how to handle root device: %s" % root) - self.disk = re.split('[0-9]', self.root)[0] + out.success(root) - out.success('%s' % self.root) + disk_file = re.split('[0-9]', root)[0] + device = parted.Device(disk_file) + self.disk = parted.Disk(device) def _read_fstable(self, f): + if not os.path.isfile(f): raise FatalError("Unable to open: `%s'. File is missing." % f) + FileSystemEntry = namedtuple('FileSystemEntry', + 'dev mpoint fs opts freq passno') with open(f) as table: for line in iter(table): entry = line.split('#')[0].strip().split() if len(entry) != 6: continue - yield self._FileSystemEntry(*entry) + yield FileSystemEntry(*entry) def _get_root_partition(self): for entry in self._read_fstable('/etc/fstab'): @@ -98,7 +114,7 @@ class BundleVolume(): return True return False - def _mount_options(self, device): + def _get_mount_options(self, device): for entry in self._read_fstable('/proc/mounts'): if not entry.dev.startswith('/'): continue @@ -106,62 +122,66 @@ class BundleVolume(): if os.path.realpath(entry.dev) == os.path.realpath(device): return entry - return + return None - def _create_partition_table(self, src_disk, dest_file): + def _create_partition_table(self, image): - if src_disk.type != 'msdos': + if self.disk.type != 'msdos': raise FatalError('Only msdos partition tables are supported') - first_sector = src_disk.getPrimaryPartitions()[0].geometry.start - # Copy the MBR and the space between the MBR and the first partition. # In Grub version 1 Stage 1.5 is located there. - first_sector = src_disk.getPrimaryPartitions()[0].geometry.start + first_sector = self.disk.getPrimaryPartitions()[0].geometry.start - dd('if=%s' % src_disk.device.path, 'of=%s' % dest_file, - 'bs=%d' % src_disk.device.sectorSize, + dd('if=%s' % self.disk.device.path, 'of=%s' % image, + 'bs=%d' % self.disk.device.sectorSize, 'count=%d' % first_sector, 'conv=notrunc') # Create the Extended boot records (EBRs) in the image - extended = src_disk.getExtendedPartition() + extended = self.disk.getExtendedPartition() if not extended: return # Extended boot records precede the logical partitions they describe - logical = src_disk.getLogicalPartitions() + logical = self.disk.getLogicalPartitions() start = extended.geometry.start for i in range(len(logical)): end = logical[i].geometry.start - 1 - dd('if=%s' % src_disk.device.path, 'of=%s' % dest_file, + dd('if=%s' % self.disk.device.path, 'of=%s' % image, 'count=%d' % (end - start + 1), 'conv=notrunc', 'seek=%d' % start, 'skip=%d' % start) start = logical[i].geometry.end + 1 - def _shrink_partitions(self, src_disk, image_file): + def _get_partitions(self, disk): + Partition = namedtuple('Partition', 'num start end type fs') partitions = [] - new_end = 0 + for p in disk.partitions: + num = p.number + start = p.geometry.start + end = p.geometry.end + ptype = p.type + fs = p.fileSystem.type if p.fileSystem is not None else '' + partitions.append(Partition(num, start, end, ptype, fs)) + + return partitions + + def _shrink_partitions(self, image): - image_dev = parted.Device(image_file) + new_end = self.disk.device.getLength() + + image_dev = parted.Device(image) image_disk = parted.Disk(image_dev) is_extended = lambda p: p.type == parted.PARTITION_EXTENDED is_logical = lambda p: p.type == parted.PARTITION_LOGICAL - partitions = [] - for p in src_disk.partitions: - g = p.geometry - f = p.fileSystem - partitions.append(self._Partition(p.number, g.start, g.end, - p.type, f.type if f is not None else '', - self._mount_options(p.path))) + partitions = self._get_partitions(self.disk) last = partitions[-1] - new_end = src_disk.device.getLength() if last.fs == 'linux-swap(v1)': MB = 2 ** 20 - size = (last.end - last.start + 1) * src_disk.device.sectorSize + size = (last.end - last.start + 1) * self.disk.device.sectorSize self.meta['SWAP'] = "%d:%s" % (last.num, (size + MB - 1) // MB) image_disk.deletePartition( @@ -180,22 +200,19 @@ class BundleVolume(): # Leave 2048 blocks at the end new_end = last.end + 2048 - if last.mopts.mpoint: - stat = os.statvfs(last.mopts.mpoint) - # Shrink the last partition. The new size should be the - # size of the occupied blocks + mount_options = self._get_mount_options( + self.disk.getPartitionBySector(last.start).path) + if mount_options is not None: + stat = os.statvfs(mount_options.mpoint) + # Shrink the last partition. The new size should be the size of the + # occupied blocks blcks = stat.f_blocks - stat.f_bavail - new_size = (blcks * stat.f_frsize) // src_disk.device.sectorSize + new_size = (blcks * stat.f_frsize) // self.disk.device.sectorSize # Add 10% just to be on the safe side part_end = last.start + (new_size * 11) // 10 # Alighn to 2048 part_end = ((part_end + 2047) // 2048) * 2048 - last = last._replace(end=part_end) - partitions[-1] = last - - # Leave 2048 blocks at the end. - new_end = new_size + 2048 image_disk.setPartitionGeometry( image_disk.getPartitionBySector(last.start), @@ -203,6 +220,15 @@ class BundleVolume(): start=last.start, end=last.end) image_disk.commit() + # Parted may have changed this for better alignment + part_end = image_disk.getPartitionBySector(last.start).geometry.end + last = last._replace(end=part_end) + partitions[-1] = last + + # Leave 2048 blocks at the end. + new_end = new_size + 2048 + + if last.type == parted.PARTITION_LOGICAL: # Fix the extended partition extended = disk.getExtendedPartition() @@ -212,52 +238,117 @@ class BundleVolume(): ext.geometry.start, end=last.end) image_disk.commit() - # Check if the available space is enough to host the image - location = os.path.dirname(image_file) - size = (new_end + 1) * src_disk.device.sectorSize - self.out.output("Examining available space in %s..." % location, False) - stat = os.statvfs(location) - available = stat.f_bavail * stat.f_frsize - if available <= size: - raise FatalError('Not enough space in %s to host the image' % \ - location) - self.out.success("sufficient") + return new_end - return partitions + def _map_partition(self, dev, num, start, end): + name = os.path.basename(dev) + tablefd, table = tempfile.mkstemp() + try: + size = end - start + 1 + os.write(tablefd, "0 %d linear %s %d" % (size, dev, start)) + dmsetup('create', "%sp%d" % (name, num), table) + finally: + os.unlink(table) - def _fill_partitions(self, src_disk, image, partitions): - pass + return "/dev/mapper/%sp%d" % (name, num) - def create_image(self): + def _unmap_partition(self, dev): + if not os.path.exists(dev): + return - image_file = '/mnt/%s.diskdump' % uuid.uuid4().hex + dmsetup('remove', dev.split('/dev/mapper/')[1]) + time.sleep(0.1) - src_dev = parted.Device(self.disk) + def _mount(self, target, devs): + + devs.sort(key=lambda d: d[1]) + for dev, mpoint in devs: + absmpoint = os.path.abspath(target + mpoint) + if not os.path.exists(absmpoint): + os.makedirs(absmpoint) + mount(dev, absmpoint) + + def _umount_all(self, target): + mpoints = [] + for entry in self._read_fstable('/proc/mounts'): + if entry.mpoint.startswith(os.path.abspath(target)): + mpoints.append(entry.mpoint) + + mpoints.sort() + for mpoint in reversed(mpoints): + umount(mpoint) + + def _create_filesystems(self, image): + + partitions = self._get_partitions(parted.Disk(parted.Device(image))) + filesystems = {} + for p in self.disk.partitions: + filesystems[p.number] = self._get_mount_options(p.path) + + unmounted = filter(lambda p: filesystems[p.num] is None, partitions) + mounted = filter(lambda p: filesystems[p.num] is not None, partitions) + + # For partitions that are not mounted right now, we can simply dd them + # into the image. + for p in unmounted: + dd('if=%s' % self.disk.device.path, 'of=%s' % image, + 'count=%d' % (p.end - p.start + 1), 'conv=notrunc', + 'seek=%d' % p.start, 'skip=%d' % p.start) + + loop = str(losetup('-f', '--show', image)).strip() + mapped = {} + try: + for p in mounted: + i = p.num + mapped[i] = self._map_partition(loop, i, p.start, p.end) + + # Create the file systems + for i, dev in mapped.iteritems(): + fs = filesystems[i].fs + self.out.output('Creating %s filesystem on partition %d ... ' % + (fs, i), False) + get_command('mkfs.%s' % fs)(*(MKFS_OPTS[fs] + [dev])) + self.out.success('done') + + target = tempfile.mkdtemp() + try: + absmpoints = self._mount(target, + [(mapped[i], filesystems[i].mpoint) for i in mapped.keys()] + ) + + finally: + self._umount_all(target) + os.rmdir(target) + finally: + for dev in mapped.values(): + self._unmap_partition(dev) + losetup('-d', loop) + + def create_image(self): - size = src_dev.getLength() * src_dev.sectorSize + image = '/mnt/%s.diskdump' % uuid.uuid4().hex + + disk_size = self.disk.device.getLength() * self.disk.device.sectorSize # Create sparse file to host the image - truncate("-s", "%d" % size, image_file) + truncate("-s", "%d" % disk_size, image) + + self._create_partition_table(image) + end_sector = self._shrink_partitions(image) - src_disk = parted.Disk(src_dev) - self._create_partition_table(src_disk, image_file) - partitions = self._shrink_partitions(src_disk, image_file) - self._fill_partitions(src_disk, image_file, partitions) + # Check if the available space is enough to host the image + dirname = os.path.dirname(image) + size = (end_sector + 1) * self.disk.device.sectorSize + self.out.output("Examining available space in %s ..." % dirname, False) + stat = os.statvfs(dirname) + available = stat.f_bavail * stat.f_frsize + if available <= size: + raise FatalError('Not enough space in %s to host the image' % + dirname) + self.out.success("sufficient") - return image_file + self._create_filesystems(image) -# unmounted = filter(lambda p: not p.mopts.mpoint, partitions) -# mounted = filter(lambda p: p.mopts.mpoint, partitions) -# -# for p in unmounted: -# dd('if=%s' % src_dev.path, 'of=%s' % img_dev.path, -# 'count=%d' % (p.end - p.start + 1), 'conv=notrunc', -# 'seek=%d' % p.start, 'skip=%d' % p.start) -# -# partition_devices = create_devices(dest, partitions) -# -# mounted.sort(key=lambda p: p.mopts.mpoint) -# -# return img + return image # vim: set sta sts=4 shiftwidth=4 sw=4 et ai : diff --git a/image_creator/disk.py b/image_creator/disk.py index 3e59830..b662c07 100644 --- a/image_creator/disk.py +++ b/image_creator/disk.py @@ -79,24 +79,6 @@ class Disk(object): self._add_cleanup(losetup, '-d', loop) return loop - def _map_partition(self, dev, index, start, end): - name = "%sp%d" % (os.path.basename(dev), index) - tablefd, table = tempfile.mkstemp() - try: - size = end - start + 1 - os.write(tablefd, "0 %d linear %s %d" % (start, dev, size)) - dmsetup('create', name, table) - finally: - os.unlink(table) - - def _unmap_partition(self, dev, index): - name = "%sp%d" % (os.path.basename(dev), index) - if not os.path.exists("/dev/mapper/%s" % name): - return - - dmsetup('remove', name) - time.sleep(0.5) - def _dir_to_disk(self): if self.source == '/': bundle = BundleVolume(self.out, self.meta) @@ -231,7 +213,7 @@ class DiskDevice(object): self.progressbar.success('done') self.progressbar = None - self.out.output('Inspecting Operating System...', False) + self.out.output('Inspecting Operating System ...', False) roots = self.g.inspect_os() if len(roots) == 0: raise FatalError("No operating system found")