Statistics
| Branch: | Tag: | Revision:

root / image_creator / bundle_volume.py @ 63af9c37

History | View | Annotate | Download (19.9 kB)

1
# -*- coding: utf-8 -*-
2
#
3
# Copyright 2012 GRNET S.A. All rights reserved.
4
#
5
# Redistribution and use in source and binary forms, with or
6
# without modification, are permitted provided that the following
7
# conditions are met:
8
#
9
#   1. Redistributions of source code must retain the above
10
#      copyright notice, this list of conditions and the following
11
#      disclaimer.
12
#
13
#   2. Redistributions in binary form must reproduce the above
14
#      copyright notice, this list of conditions and the following
15
#      disclaimer in the documentation and/or other materials
16
#      provided with the distribution.
17
#
18
# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
19
# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
20
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
21
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
22
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
25
# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
26
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
27
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
28
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29
# POSSIBILITY OF SUCH DAMAGE.
30
#
31
# The views and conclusions contained in the software and
32
# documentation are those of the authors and should not be
33
# interpreted as representing official policies, either expressed
34
# or implied, of GRNET S.A.
35

    
36
"""This module hosts the code that performes the host bundling operation. By
37
using the create_image method of the BundleVolume class the user can create an
38
image out of the running system.
39
"""
40

    
41
import os
42
import re
43
import tempfile
44
import uuid
45
from collections import namedtuple
46

    
47
import parted
48

    
49
from image_creator.rsync import Rsync
50
from image_creator.util import get_command
51
from image_creator.util import FatalError
52
from image_creator.util import try_fail_repeat
53
from image_creator.util import free_space
54
from image_creator.gpt import GPTPartitionTable
55

    
56
findfs = get_command('findfs')
57
dd = get_command('dd')
58
dmsetup = get_command('dmsetup')
59
losetup = get_command('losetup')
60
mount = get_command('mount')
61
umount = get_command('umount')
62
blkid = get_command('blkid')
63
tune2fs = get_command('tune2fs')
64

    
65
MKFS_OPTS = {'ext2': ['-F'],
66
             'ext3': ['-F'],
67
             'ext4': ['-F'],
68
             'reiserfs': ['-ff'],
69
             'btrfs': [],
70
             'minix': [],
71
             'xfs': ['-f'],
72
             'jfs': ['-f'],
73
             'ntfs': ['-F'],
74
             'msdos': [],
75
             'vfat': []}
76

    
77

    
78
class BundleVolume(object):
79
    """This class can be used to create an image out of the running system"""
80

    
81
    def __init__(self, out, meta, tmp=None):
82
        """Create an instance of the BundleVolume class."""
83
        self.out = out
84
        self.meta = meta
85
        self.tmp = tmp
86

    
87
        self.out.output('Searching for root device ...', False)
88
        root = self._get_root_partition()
89

    
90
        if root.startswith("UUID=") or root.startswith("LABEL="):
91
            root = findfs(root).stdout.strip()
92

    
93
        if not re.match('/dev/[hsv]d[a-z][1-9]*$', root):
94
            raise FatalError("Don't know how to handle root device: %s" % root)
95

    
96
        out.success(root)
97

    
98
        disk_file = re.split('[0-9]', root)[0]
99
        device = parted.Device(disk_file)
100
        self.disk = parted.Disk(device)
101

    
102
    def _read_fstable(self, f):
103
        """Use this generator to iterate over the lines of and fstab file"""
104

    
105
        if not os.path.isfile(f):
106
            raise FatalError("Unable to open: `%s'. File is missing." % f)
107

    
108
        FileSystemTableEntry = namedtuple('FileSystemTableEntry',
109
                                          'dev mpoint fs opts freq passno')
110
        with open(f) as table:
111
            for line in iter(table):
112
                entry = line.split('#')[0].strip().split()
113
                if len(entry) != 6:
114
                    continue
115
                yield FileSystemTableEntry(*entry)
116

    
117
    def _get_root_partition(self):
118
        """Return the fstab entry accosiated with the root filesystem"""
119
        for entry in self._read_fstable('/etc/fstab'):
120
            if entry.mpoint == '/':
121
                return entry.dev
122

    
123
        raise FatalError("Unable to find root device in /etc/fstab")
124

    
125
    def _is_mpoint(self, path):
126
        """Check if a directory is currently a mount point"""
127
        for entry in self._read_fstable('/proc/mounts'):
128
            if entry.mpoint == path:
129
                return True
130
        return False
131

    
132
    def _get_mount_options(self, device):
133
        """Return the mount entry associated with a mounted device"""
134
        for entry in self._read_fstable('/proc/mounts'):
135
            if not entry.dev.startswith('/'):
136
                continue
137

    
138
            if os.path.realpath(entry.dev) == os.path.realpath(device):
139
                return entry
140

    
141
        return None
142

    
143
    def _create_partition_table(self, image):
144
        """Copy the partition table of the host system into the image"""
145

    
146
        # Copy the MBR and the space between the MBR and the first partition.
147
        # In msdos partition tables Grub Stage 1.5 is located there.
148
        # In gpt partition tables the Primary GPT Header is there.
149
        first_sector = self.disk.getPrimaryPartitions()[0].geometry.start
150

    
151
        dd('if=%s' % self.disk.device.path, 'of=%s' % image,
152
           'bs=%d' % self.disk.device.sectorSize,
153
           'count=%d' % first_sector, 'conv=notrunc')
154

    
155
        if self.disk.type == 'gpt':
156
            # Copy the Secondary GPT Header
157
            table = GPTPartitionTable(self.disk.device.path)
158
            dd('if=%s' % self.disk.device.path, 'of=%s' % image,
159
               'bs=%d' % self.disk.device.sectorSize, 'conv=notrunc',
160
               'seek=%d' % table.primary.last_usable_lba,
161
               'skip=%d' % table.primary.last_usable_lba)
162

    
163
        # Create the Extended boot records (EBRs) in the image
164
        extended = self.disk.getExtendedPartition()
165
        if not extended:
166
            return
167

    
168
        # Extended boot records precede the logical partitions they describe
169
        logical = self.disk.getLogicalPartitions()
170
        start = extended.geometry.start
171
        for i in range(len(logical)):
172
            end = logical[i].geometry.start - 1
173
            dd('if=%s' % self.disk.device.path, 'of=%s' % image,
174
               'count=%d' % (end - start + 1), 'conv=notrunc',
175
               'seek=%d' % start, 'skip=%d' % start)
176
            start = logical[i].geometry.end + 1
177

    
178
    def _get_partitions(self, disk):
179
        """Returns a list with the partitions of the provided disk"""
180
        Partition = namedtuple('Partition', 'num start end type fs')
181

    
182
        partitions = []
183
        for p in disk.partitions:
184
            num = p.number
185
            start = p.geometry.start
186
            end = p.geometry.end
187
            ptype = p.type
188
            fs = p.fileSystem.type if p.fileSystem is not None else ''
189
            partitions.append(Partition(num, start, end, ptype, fs))
190

    
191
        return partitions
192

    
193
    def _shrink_partitions(self, image):
194
        """Remove the last partition of the image if it is a swap partition and
195
        shrink the partition before that. Make sure it can still host all the
196
        files the corresponding host file system hosts
197
        """
198
        new_end = self.disk.device.length
199

    
200
        image_disk = parted.Disk(parted.Device(image))
201

    
202
        is_extended = lambda p: p.type == parted.PARTITION_EXTENDED
203
        is_logical = lambda p: p.type == parted.PARTITION_LOGICAL
204

    
205
        partitions = self._get_partitions(self.disk)
206

    
207
        last = partitions[-1]
208
        if last.fs == 'linux-swap(v1)':
209
            MB = 2 ** 20
210
            size = (last.end - last.start + 1) * self.disk.device.sectorSize
211
            self.meta['SWAP'] = "%d:%s" % (last.num, (size + MB - 1) // MB)
212

    
213
            image_disk.deletePartition(
214
                image_disk.getPartitionBySector(last.start))
215
            image_disk.commitToDevice()
216

    
217
            if is_logical(last) and last.num == 5:
218
                extended = image_disk.getExtendedPartition()
219
                image_disk.deletePartition(extended)
220
                image_disk.commitToDevice()
221
                partitions.remove(filter(is_extended, partitions)[0])
222

    
223
            partitions.remove(last)
224
            last = partitions[-1]
225

    
226
            new_end = last.end
227

    
228
        mount_options = self._get_mount_options(
229
            self.disk.getPartitionBySector(last.start).path)
230
        if mount_options is not None:
231
            stat = os.statvfs(mount_options.mpoint)
232
            # Shrink the last partition. The new size should be the size of the
233
            # occupied blocks
234
            blcks = stat.f_blocks - stat.f_bavail
235
            new_size = (blcks * stat.f_frsize) // self.disk.device.sectorSize
236

    
237
            # Add 10% just to be on the safe side
238
            part_end = last.start + (new_size * 11) // 10
239
            # Align to 2048
240
            part_end = ((part_end + 2047) // 2048) * 2048
241

    
242
            # Make sure the partition starts where the old partition started.
243
            constraint = parted.Constraint(device=image_disk.device)
244
            constraint.startRange = parted.Geometry(device=image_disk.device,
245
                                                    start=last.start, length=1)
246

    
247
            image_disk.setPartitionGeometry(
248
                image_disk.getPartitionBySector(last.start), constraint,
249
                start=last.start, end=part_end)
250
            image_disk.commitToDevice()
251

    
252
            # Parted may have changed this for better alignment
253
            part_end = image_disk.getPartitionBySector(last.start).geometry.end
254
            last = last._replace(end=part_end)
255
            partitions[-1] = last
256

    
257
            new_end = part_end
258

    
259
            if last.type == parted.PARTITION_LOGICAL:
260
                # Fix the extended partition
261
                image_disk.minimizeExtendedPartition()
262

    
263
        return (new_end, self._get_partitions(image_disk))
264

    
265
    def _map_partition(self, dev, num, start, end):
266
        """Map a partition into a block device using the device mapper"""
267
        name = os.path.basename(dev) + "_" + uuid.uuid4().hex
268
        tablefd, table = tempfile.mkstemp()
269
        try:
270
            try:
271
                size = end - start + 1
272
                os.write(tablefd, "0 %d linear %s %d" % (size, dev, start))
273
            finally:
274
                os.close(tablefd)
275
            dmsetup('create', "%sp%d" % (name, num), table)
276
        finally:
277
            os.unlink(table)
278

    
279
        return "/dev/mapper/%sp%d" % (name, num)
280

    
281
    def _unmap_partition(self, dev):
282
        """Unmap a previously mapped partition"""
283
        if not os.path.exists(dev):
284
            return
285

    
286
        try_fail_repeat(dmsetup, 'remove', dev.split('/dev/mapper/')[1])
287

    
288
    def _mount(self, target, devs):
289
        """Mount a list of filesystems in mountpoints relative to target"""
290
        devs.sort(key=lambda d: d[1])
291
        for dev, mpoint, options in devs:
292
            absmpoint = os.path.abspath(target + mpoint)
293
            if not os.path.exists(absmpoint):
294
                os.makedirs(absmpoint)
295

    
296
            if len(options) > 0:
297
                mount(dev, absmpoint, '-o', ",".join(options))
298
            else:
299
                mount(dev, absmpoint)
300

    
301
    def _umount_all(self, target):
302
        """Unmount all filesystems that are mounted under the directory target
303
        """
304
        mpoints = []
305
        for entry in self._read_fstable('/proc/mounts'):
306
            if entry.mpoint.startswith(os.path.abspath(target)):
307
                    mpoints.append(entry.mpoint)
308

    
309
        mpoints.sort()
310
        for mpoint in reversed(mpoints):
311
            try_fail_repeat(umount, mpoint)
312

    
313
    def _to_exclude(self):
314
        """Find which directories to exclude during the image copy. This is
315
        accompliced by checking which directories serve as mount points for
316
        virtual file systems
317
        """
318
        excluded = ['/tmp', '/var/tmp']
319
        if self.tmp is not None:
320
            excluded.append(self.tmp)
321
        local_filesystems = MKFS_OPTS.keys() + ['rootfs']
322
        for entry in self._read_fstable('/proc/mounts'):
323
            if entry.fs in local_filesystems:
324
                continue
325

    
326
            mpoint = entry.mpoint
327
            if mpoint in excluded:
328
                continue
329

    
330
            descendants = filter(
331
                lambda p: p.startswith(mpoint + '/'), excluded)
332
            if len(descendants):
333
                for d in descendants:
334
                    excluded.remove(d)
335
                excluded.append(mpoint)
336
                continue
337

    
338
            dirname = mpoint
339
            basename = ''
340
            found_ancestor = False
341
            while dirname != '/':
342
                (dirname, basename) = os.path.split(dirname)
343
                if dirname in excluded:
344
                    found_ancestor = True
345
                    break
346

    
347
            if not found_ancestor:
348
                excluded.append(mpoint)
349

    
350
        return excluded
351

    
352
    def _replace_uuids(self, target, new_uuid):
353
        """Replace UUID references in various files. This is needed after
354
        copying system files of the host into a new filesystem
355
        """
356

    
357
        files = ['/etc/fstab',
358
                 '/boot/grub/grub.cfg',
359
                 '/boot/grub/menu.lst',
360
                 '/boot/grub/grub.conf']
361

    
362
        orig = {}
363
        for p in self.disk.partitions:
364
            if p.number in new_uuid.keys():
365
                orig[p.number] = \
366
                    blkid('-s', 'UUID', '-o', 'value', p.path).stdout.strip()
367

    
368
        for f in map(lambda f: target + f, files):
369
            if not os.path.exists(f):
370
                continue
371

    
372
            with open(f, 'r') as src:
373
                lines = src.readlines()
374
            with open(f, 'w') as dest:
375
                for line in lines:
376
                    for i, uuid in new_uuid.items():
377
                        line = re.sub(orig[i], uuid, line)
378
                    dest.write(line)
379

    
380
    def _create_filesystems(self, image, partitions):
381
        """Fill the image with data. Host file systems that are not currently
382
        mounted are binary copied into the image. For mounted file systems, a
383
        file system level copy is performed.
384
        """
385

    
386
        filesystem = {}
387
        orig_dev = {}
388
        for p in self.disk.partitions:
389
            filesystem[p.number] = self._get_mount_options(p.path)
390
            orig_dev[p.number] = p.path
391

    
392
        unmounted = filter(lambda p: filesystem[p.num] is None, partitions)
393
        mounted = filter(lambda p: filesystem[p.num] is not None, partitions)
394

    
395
        # For partitions that are not mounted right now, we can simply dd them
396
        # into the image.
397
        for p in unmounted:
398
            self.out.output('Cloning partition %d ... ' % p.num, False)
399
            dd('if=%s' % self.disk.device.path, 'of=%s' % image,
400
               'count=%d' % (p.end - p.start + 1), 'conv=notrunc',
401
               'seek=%d' % p.start, 'skip=%d' % p.start)
402
            self.out.success("done")
403

    
404
        loop = str(losetup('-f', '--show', image)).strip()
405

    
406
        # Recreate mounted file systems
407
        mapped = {}
408
        try:
409
            for p in mounted:
410
                i = p.num
411
                mapped[i] = self._map_partition(loop, i, p.start, p.end)
412

    
413
            new_uuid = {}
414
            # Create the file systems
415
            for i, dev in mapped.iteritems():
416
                fs = filesystem[i].fs
417
                self.out.output('Creating %s filesystem on partition %d ... ' %
418
                                (fs, i), False)
419
                get_command('mkfs.%s' % fs)(*(MKFS_OPTS[fs] + [dev]))
420

    
421
                # For ext[234] enable the default mount options
422
                if re.match('^ext[234]$', fs):
423
                    mopts = filter(
424
                        lambda p: p.startswith('Default mount options:'),
425
                        tune2fs('-l', orig_dev[i]).splitlines()
426
                    )[0].split(':')[1].strip().split()
427

    
428
                    if not (len(mopts) == 1 and mopts[0] == '(none)'):
429
                        for opt in mopts:
430
                            tune2fs('-o', opt, dev)
431

    
432
                self.out.success('done')
433
                new_uuid[i] = blkid(
434
                    '-s', 'UUID', '-o', 'value', dev).stdout.strip()
435

    
436
            target = tempfile.mkdtemp()
437
            devs = []
438
            for i in mapped.keys():
439
                fs = filesystem[i].fs
440
                mpoint = filesystem[i].mpoint
441
                opts = []
442
                for opt in filesystem[i].opts.split(','):
443
                    if opt in ('acl', 'user_xattr'):
444
                        opts.append(opt)
445
                devs.append((mapped[i], mpoint, opts))
446
            try:
447
                self._mount(target, devs)
448

    
449
                excluded = self._to_exclude()
450

    
451
                rsync = Rsync(self.out)
452

    
453
                for excl in excluded + [image]:
454
                    rsync.exclude(excl)
455

    
456
                rsync.archive().hard_links().xattrs().sparse().acls()
457
                rsync.run('/', target, 'host', 'temporary image')
458

    
459
                # Create missing mountpoints. Since they are mountpoints, we
460
                # cannot determine the ownership and the mode of the real
461
                # directory. Make them inherit those properties from their
462
                # parent dir
463
                for excl in excluded:
464
                    dirname = os.path.dirname(excl)
465
                    stat = os.stat(dirname)
466
                    os.mkdir(target + excl)
467
                    os.chmod(target + excl, stat.st_mode)
468
                    os.chown(target + excl, stat.st_uid, stat.st_gid)
469

    
470
                # /tmp and /var/tmp are special cases. We exclude then even if
471
                # they aren't mountpoints. Restore their permissions.
472
                for excl in ('/tmp', '/var/tmp'):
473
                    if self._is_mpoint(excl):
474
                        os.chmod(target + excl, 041777)
475
                        os.chown(target + excl, 0, 0)
476
                    else:
477
                        stat = os.stat(excl)
478
                        os.chmod(target + excl, stat.st_mode)
479
                        os.chown(target + excl, stat.st_uid, stat.st_gid)
480

    
481
                # We need to replace the old UUID referencies with the new
482
                # ones in grub configuration files and /etc/fstab for file
483
                # systems that have been recreated.
484
                self._replace_uuids(target, new_uuid)
485

    
486
            finally:
487
                self._umount_all(target)
488
                os.rmdir(target)
489
        finally:
490
            for dev in mapped.values():
491
                self._unmap_partition(dev)
492
            losetup('-d', loop)
493

    
494
    def create_image(self, image):
495
        """Given an image filename, this method will create an image out of the
496
        running system.
497
        """
498

    
499
        size = self.disk.device.length * self.disk.device.sectorSize
500

    
501
        # Create sparse file to host the image
502
        fd = os.open(image, os.O_WRONLY | os.O_CREAT)
503
        try:
504
            os.ftruncate(fd, size)
505
        finally:
506
            os.close(fd)
507

    
508
        self._create_partition_table(image)
509
        end_sector, partitions = self._shrink_partitions(image)
510

    
511
        if self.disk.type == 'gpt':
512
            old_size = size
513
            size = (end_sector + 1) * self.disk.device.sectorSize
514
            ptable = GPTPartitionTable(image)
515
            size = ptable.shrink(size, old_size)
516
        else:
517
            # Alighn to 2048
518
            end_sector = ((end_sector + 2047) // 2048) * 2048
519
            size = (end_sector + 1) * self.disk.device.sectorSize
520

    
521
        # Truncate image to the new size.
522
        fd = os.open(image, os.O_RDWR)
523
        try:
524
            os.ftruncate(fd, size)
525
        finally:
526
            os.close(fd)
527

    
528
        # Check if the available space is enough to host the image
529
        dirname = os.path.dirname(image)
530
        self.out.output("Examining available space ...", False)
531
        if free_space(dirname) <= size:
532
            raise FatalError("Not enough space under %s to host the temporary "
533
                             "image" % dirname)
534
        self.out.success("sufficient")
535

    
536
        self._create_filesystems(image, partitions)
537

    
538
        return image
539

    
540
# vim: set sta sts=4 shiftwidth=4 sw=4 et ai :