From: Nikos Skalkotos Date: Sun, 23 Dec 2012 19:12:55 +0000 (+0200) Subject: In bundle volume copy host files with rsync X-Git-Tag: v0.2~29 X-Git-Url: https://code.grnet.gr/git/snf-image-creator/commitdiff_plain/74149d07caa5363494b071483e55af861b7d953f In bundle volume copy host files with rsync --- diff --git a/image_creator/bundle_volume.py b/image_creator/bundle_volume.py index ee93fc8..fde1e00 100644 --- a/image_creator/bundle_volume.py +++ b/image_creator/bundle_volume.py @@ -40,6 +40,7 @@ from collections import namedtuple import parted +from image_creator.rsync import Rsync from image_creator.util import get_command from image_creator.util import FatalError @@ -92,14 +93,14 @@ class BundleVolume(): if not os.path.isfile(f): raise FatalError("Unable to open: `%s'. File is missing." % f) - FileSystemEntry = namedtuple('FileSystemEntry', + FileSystemTableEntry = namedtuple('FileSystemTableEntry', 'dev mpoint fs opts freq passno') with open(f) as table: for line in iter(table): entry = line.split('#')[0].strip().split() if len(entry) != 6: continue - yield FileSystemEntry(*entry) + yield FileSystemTableEntry(*entry) def _get_root_partition(self): for entry in self._read_fstable('/etc/fstab'): @@ -202,7 +203,7 @@ class BundleVolume(): mount_options = self._get_mount_options( self.disk.getPartitionBySector(last.start).path) - if mount_options is not None: + if mount_options is not None: stat = os.statvfs(mount_options.mpoint) # Shrink the last partition. The new size should be the size of the # occupied blocks @@ -228,7 +229,6 @@ class BundleVolume(): # Leave 2048 blocks at the end. new_end = new_size + 2048 - if last.type == parted.PARTITION_LOGICAL: # Fix the extended partition extended = disk.getExtendedPartition() @@ -273,13 +273,46 @@ class BundleVolume(): for entry in self._read_fstable('/proc/mounts'): if entry.mpoint.startswith(os.path.abspath(target)): mpoints.append(entry.mpoint) - + mpoints.sort() for mpoint in reversed(mpoints): umount(mpoint) + def _to_exclude(self): + excluded = ['/tmp'] + local_filesystems = MKFS_OPTS.keys() + ['rootfs'] + for entry in self._read_fstable('/proc/mounts'): + if entry.fs in local_filesystems: + continue + + mpoint = entry.mpoint + if mpoint in excluded: + continue + + descendants = filter(lambda p: p.startswith(mpoint + '/'), + excluded) + if len(descendants): + for d in descendants: + excluded.remove(d) + excluded.append(mpoint) + continue + + dirname = mpoint + basename = '' + found_ancestor = False + while dirname != '/': + (dirname, basename) = os.path.split(dirname) + if dirname in excluded: + found_ancestor = True + break + + if not found_ancestor: + excluded.append(mpoint) + + return map(lambda d: d + "/*", excluded) + def _create_filesystems(self, image): - + partitions = self._get_partitions(parted.Disk(parted.Device(image))) filesystems = {} for p in self.disk.partitions: @@ -299,7 +332,7 @@ class BundleVolume(): mapped = {} try: for p in mounted: - i = p.num + i = p.num mapped[i] = self._map_partition(loop, i, p.start, p.end) # Create the file systems @@ -315,6 +348,10 @@ class BundleVolume(): absmpoints = self._mount(target, [(mapped[i], filesystems[i].mpoint) for i in mapped.keys()] ) + exclude = self._to_exclude() + [image] + rsync = Rsync('/', target, exclude) + msg = "Copying host files into the image" + rsync.archive().run(self.out, msg) finally: self._umount_all(target) diff --git a/image_creator/rsync.py b/image_creator/rsync.py new file mode 100644 index 0000000..22d3ad7 --- /dev/null +++ b/image_creator/rsync.py @@ -0,0 +1,71 @@ +# Copyright 2012 GRNET S.A. All rights reserved. +# +# Redistribution and use in source and binary forms, with or +# without modification, are permitted provided that the following +# conditions are met: +# +# 1. Redistributions of source code must retain the above +# copyright notice, this list of conditions and the following +# disclaimer. +# +# 2. Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials +# provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS +# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF +# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED +# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN +# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# +# The views and conclusions contained in the software and +# documentation are those of the authors and should not be +# interpreted as representing official policies, either expressed +# or implied, of GRNET S.A. + +import subprocess + + +class Rsync: + def __init__(self, src, dest, exclude=[]): + self.src = src + self.dest = dest + self.exclude = exclude + self.options = ['-v'] + + def archive(self): + self.options.append('-a') + return self + + def run(self, out, msg): + cmd = [] + cmd.append('rsync') + cmd.extend(self.options) + for i in self.exclude: + cmd.extend(['--exclude', i]) + + dry_run = subprocess.Popen(cmd + ['-n', self.src, self.dest], + shell=False, stdout=subprocess.PIPE, + bufsize=0) + print "%r" % (cmd + ['-n', self.src, self.dest]) + total = 0 + for line in iter(dry_run.stdout.readline, b''): + total += 1 + + progress = out.Progress(total, msg) + run = subprocess.Popen(cmd + [self.src, self.dest], shell=False, + stdout=subprocess.PIPE, bufsize=0) + for line in iter(run.stdout.readline, b''): + progress.next() + + progress.success('done') + +# vim: set sta sts=4 shiftwidth=4 sw=4 et ai :