Revision f5174d2c

b/image_creator/dialog_main.py
48 48
from image_creator.output.dialog import GaugeOutput
49 49
from image_creator.output.composite import CompositeOutput
50 50
from image_creator.disk import Disk
51
from image_creator.os_type import os_cls
52 51
from image_creator.dialog_wizard import wizard
53 52
from image_creator.dialog_menu import main_menu
54 53
from image_creator.dialog_util import SMALL_WIDTH, WIDTH, confirm_exit, \
55 54
    Reset, update_background_title
56 55

  
57 56

  
58
def image_creator(d, media, out, tmp):
57
def create_image(d, media, out, tmp):
59 58

  
60 59
    d.setBackgroundTitle('snf-image-creator')
61 60

  
......
71 70
    signal.signal(signal.SIGTERM, signal_handler)
72 71
    try:
73 72
        snapshot = disk.snapshot()
74
        dev = disk.get_device(snapshot)
73
        image = disk.get_image(snapshot)
75 74

  
75
        out.output("Collecting image metadata...")
76 76
        metadata = {}
77
        for (key, value) in dev.meta.items():
77
        for (key, value) in image.meta.items():
78 78
            metadata[str(key)] = str(value)
79 79

  
80
        dev.mount(readonly=True)
81
        out.output("Collecting image metadata...")
82
        cls = os_cls(dev.distro, dev.ostype)
83
        image_os = cls(dev.root, dev.g, out)
84
        dev.umount()
85

  
86
        for (key, value) in image_os.meta.items():
80
        for (key, value) in image.os.meta.items():
87 81
            metadata[str(key)] = str(value)
88 82

  
89 83
        out.success("done")
......
97 91

  
98 92
        session = {"dialog": d,
99 93
                   "disk": disk,
100
                   "snapshot": snapshot,
101
                   "device": dev,
102
                   "image_os": image_os,
94
                   "image": image,
103 95
                   "metadata": metadata}
104 96

  
105 97
        msg = "snf-image-creator detected a %s system on the input media. " \
......
107 99
              "image creation process?\n\nChoose <Wizard> to run the wizard," \
108 100
              " <Expert> to run the snf-image-creator in expert mode or " \
109 101
              "press ESC to quit the program." \
110
              % (dev.ostype if dev.ostype == dev.distro else "%s (%s)" %
111
                 (dev.ostype, dev.distro))
102
              % (image.ostype if image.ostype == image.distro else "%s (%s)" %
103
                 (image.ostype, image.distro))
112 104

  
113 105
        update_background_title(session)
114 106

  
......
226 218
                    out = CompositeOutput([log])
227 219
                    out.output("Starting %s v%s..." %
228 220
                               (parser.get_prog_name(), version))
229
                    ret = image_creator(d, media, out, options.tmp)
221
                    ret = create_image(d, media, out, options.tmp)
230 222
                    sys.exit(ret)
231 223
                except Reset:
232 224
                    log.output("Resetting everything...")
b/image_creator/dialog_menu.py
108 108

  
109 109
def upload_image(session):
110 110
    d = session["dialog"]
111
    dev = session['device']
111
    image = session['image']
112 112
    meta = session['metadata']
113
    size = dev.size
113
    size = image.size
114 114

  
115 115
    if "account" not in session:
116 116
        d.msgbox("You need to provide your ~okeanos credentials before you "
......
139 139

  
140 140
    gauge = GaugeOutput(d, "Image Upload", "Uploading...")
141 141
    try:
142
        out = dev.out
142
        out = image.out
143 143
        out.add(gauge)
144 144
        try:
145 145
            if 'checksum' not in session:
146 146
                md5 = MD5(out)
147
                session['checksum'] = md5.compute(session['snapshot'], size)
147
                session['checksum'] = md5.compute(image.device, size)
148 148

  
149 149
            kamaki = Kamaki(session['account'], out)
150 150
            try:
151 151
                # Upload image file
152
                with open(session['snapshot'], 'rb') as f:
152
                with open(image.device, 'rb') as f:
153 153
                    session["pithos_uri"] = \
154 154
                        kamaki.upload(f, size, filename,
155 155
                                      "Calculating block hashes",
......
188 188

  
189 189
def register_image(session):
190 190
    d = session["dialog"]
191
    dev = session['device']
192 191

  
193 192
    is_public = False
194 193

  
195 194
    if "account" not in session:
196 195
        d.msgbox("You need to provide your ~okeanos credentians before you "
197
                 "can register an images with cyclades",
198
                 width=SMALL_WIDTH)
196
                 "can register an images with cyclades", width=SMALL_WIDTH)
199 197
        return False
200 198

  
201 199
    if "pithos_uri" not in session:
......
233 231
    img_type = "public" if is_public else "private"
234 232
    gauge = GaugeOutput(d, "Image Registration", "Registering image...")
235 233
    try:
236
        out = dev.out
234
        out = session['image'].out
237 235
        out.add(gauge)
238 236
        try:
239 237
            out.output("Registering %s image with Cyclades..." % img_type)
......
481 479

  
482 480
def sysprep(session):
483 481
    d = session['dialog']
484
    image_os = session['image_os']
482
    image = session['image']
485 483

  
486 484
    # Is the image already shrinked?
487 485
    if 'shrinked' in session and session['shrinked']:
......
500 498
    if 'exec_syspreps' not in session:
501 499
        session['exec_syspreps'] = []
502 500

  
503
    all_syspreps = image_os.list_syspreps()
501
    all_syspreps = image.os.list_syspreps()
504 502
    # Only give the user the choice between syspreps that have not ran yet
505 503
    syspreps = [s for s in all_syspreps if s not in session['exec_syspreps']]
506 504

  
......
513 511
        choices = []
514 512
        index = 0
515 513
        for sysprep in syspreps:
516
            name, descr = image_os.sysprep_info(sysprep)
514
            name, descr = image.os.sysprep_info(sysprep)
517 515
            display_name = name.replace('-', ' ').capitalize()
518 516
            sysprep_help += "%s\n" % display_name
519 517
            sysprep_help += "%s\n" % ('-' * len(display_name))
......
536 534
            # Enable selected syspreps and disable the rest
537 535
            for i in range(len(syspreps)):
538 536
                if str(i + 1) in tags:
539
                    image_os.enable_sysprep(syspreps[i])
537
                    image.os.enable_sysprep(syspreps[i])
540 538
                    session['exec_syspreps'].append(syspreps[i])
541 539
                else:
542
                    image_os.disable_sysprep(syspreps[i])
540
                    image.os.disable_sysprep(syspreps[i])
543 541

  
544 542
            infobox = InfoBoxOutput(d, "Image Configuration")
545 543
            try:
546
                dev = session['device']
547
                dev.out.add(infobox)
544
                image.out.add(infobox)
548 545
                try:
549
                    dev.mount(readonly=False)
546
                    image.mount(readonly=False)
550 547
                    try:
551 548
                        # The checksum is invalid. We have mounted the image rw
552 549
                        if 'checksum' in session:
553 550
                            del session['checksum']
554 551

  
555 552
                        # Monitor the metadata changes during syspreps
556
                        with MetadataMonitor(session, image_os.meta):
557
                            image_os.do_sysprep()
553
                        with MetadataMonitor(session, image.os.meta):
554
                            image.os.do_sysprep()
558 555
                            infobox.finalize()
559 556

  
560 557
                        # Disable syspreps that have ran
561 558
                        for sysprep in session['exec_syspreps']:
562
                            image_os.disable_sysprep(sysprep)
559
                            image.os.disable_sysprep(sysprep)
563 560
                    finally:
564
                        dev.umount()
561
                        image.umount()
565 562
                finally:
566
                    dev.out.remove(infobox)
563
                    image.out.remove(infobox)
567 564
            finally:
568 565
                infobox.cleanup()
569 566
            break
......
572 569

  
573 570
def shrink(session):
574 571
    d = session['dialog']
575
    dev = session['device']
572
    image = session['image']
576 573

  
577 574
    shrinked = 'shrinked' in session and session['shrinked']
578 575

  
......
589 586

  
590 587
    if not d.yesno("%s\n\nDo you want to continue?" % msg, width=WIDTH,
591 588
                   height=12, title="Image Shrinking"):
592
        with MetadataMonitor(session, dev.meta):
589
        with MetadataMonitor(session, image.meta):
593 590
            infobox = InfoBoxOutput(d, "Image Shrinking", height=4)
594
            dev.out.add(infobox)
591
            image.out.add(infobox)
595 592
            try:
596
                dev.shrink()
593
                image.shrink()
597 594
                infobox.finalize()
598 595
            finally:
599
                dev.out.remove(infobox)
596
                image.out.remove(infobox)
600 597

  
601 598
        session['shrinked'] = True
602 599
        update_background_title(session)
b/image_creator/dialog_util.py
43 43

  
44 44
def update_background_title(session):
45 45
    d = session['dialog']
46
    dev = session['device']
47 46
    disk = session['disk']
47
    image = session['image']
48 48

  
49 49
    MB = 2 ** 20
50 50

  
51
    size = (dev.size + MB - 1) // MB
51
    size = (image.size + MB - 1) // MB
52 52
    shrinked = 'shrinked' in session and session['shrinked']
53 53
    postfix = " (shrinked)" if shrinked else ''
54 54

  
55 55
    title = "OS: %s, Distro: %s, Size: %dMB%s, Source: %s" % \
56
            (dev.ostype, dev.distro, size, postfix,
56
            (image.ostype, image.distro, size, postfix,
57 57
             os.path.abspath(disk.source))
58 58

  
59 59
    d.setBackgroundTitle(title)
......
128 128

  
129 129
        gauge = GaugeOutput(d, "Image Extraction", "Extracting image...")
130 130
        try:
131
            dev = session['device']
132
            out = dev.out
131
            image = session['image']
132
            out = image.out
133 133
            out.add(gauge)
134 134
            try:
135 135
                if "checksum" not in session:
136
                    size = dev.size
137 136
                    md5 = MD5(out)
138
                    session['checksum'] = md5.compute(session['snapshot'],
139
                                                      size)
137
                    session['checksum'] = md5.compute(image.device, image.size)
140 138

  
141 139
                # Extract image file
142
                dev.dump(path)
140
                image.dump(path)
143 141

  
144 142
                # Extract metadata file
145 143
                out.output("Extracting metadata file...")
b/image_creator/dialog_wizard.py
180 180

  
181 181
    name = WizardInputPage(
182 182
        "ImageName", "Image Name", "Please provide a name for the image:",
183
        title="Image Name", init=session['device'].distro)
183
        title="Image Name", init=session['image'].distro)
184 184

  
185 185
    descr = WizardInputPage(
186 186
        "ImageDescription", "Image Description",
......
232 232

  
233 233
def create_image(session):
234 234
    d = session['dialog']
235
    disk = session['disk']
236
    device = session['device']
237
    snapshot = session['snapshot']
238
    image_os = session['image_os']
235
    image = session['image']
239 236
    wizard = session['wizard']
240 237

  
241 238
    # Save Kamaki credentials
242 239
    Kamaki.save_token(wizard['Account']['auth_token'])
243 240

  
244 241
    with_progress = OutputWthProgress(True)
245
    out = disk.out
242
    out = image.out
246 243
    out.add(with_progress)
247 244
    try:
248 245
        out.clear()
249 246

  
250 247
        #Sysprep
251
        device.mount(False)
252
        image_os.do_sysprep()
253
        metadata = image_os.meta
254
        device.umount()
248
        image.mount(False)
249
        image.os.do_sysprep()
250
        metadata = image.os.meta
251
        image.umount()
255 252

  
256 253
        #Shrink
257
        size = device.shrink()
254
        size = image.shrink()
258 255
        session['shrinked'] = True
259 256
        update_background_title(session)
260 257

  
261
        metadata.update(device.meta)
258
        metadata.update(image.meta)
262 259
        metadata['DESCRIPTION'] = wizard['ImageDescription']
263 260

  
264 261
        #MD5
265 262
        md5 = MD5(out)
266
        session['checksum'] = md5.compute(snapshot, size)
263
        session['checksum'] = md5.compute(image.device, size)
267 264

  
268 265
        #Metadata
269 266
        metastring = '\n'.join(
......
278 275
            name = "%s-%s.diskdump" % (wizard['ImageName'],
279 276
                                       time.strftime("%Y%m%d%H%M"))
280 277
            pithos_file = ""
281
            with open(snapshot, 'rb') as f:
278
            with open(image.device, 'rb') as f:
282 279
                pithos_file = kamaki.upload(f, size, name,
283 280
                                            "(1/4)  Calculating block hashes",
284 281
                                            "(2/4)  Uploading missing blocks")
b/image_creator/disk.py
32 32
# or implied, of GRNET S.A.
33 33

  
34 34
from image_creator.util import get_command
35
from image_creator.util import FatalError
36 35
from image_creator.util import try_fail_repeat
37 36
from image_creator.util import free_space
38
from image_creator.gpt import GPTPartitionTable
37
from image_creator.util import FatalError
39 38
from image_creator.bundle_volume import BundleVolume
39
from image_creator.image import Image
40 40

  
41 41
import stat
42 42
import os
43 43
import tempfile
44 44
import uuid
45
import re
46
import guestfs
47 45
import shutil
48
from sendfile import sendfile
49

  
50 46

  
51 47
dd = get_command('dd')
52 48
dmsetup = get_command('dmsetup')
......
70 66
        media can be an image file, a block device or a directory.
71 67
        """
72 68
        self._cleanup_jobs = []
73
        self._devices = []
69
        self._images = []
74 70
        self.source = source
75 71
        self.out = output
76 72
        self.meta = {}
......
123 119
        program ends.
124 120
        """
125 121
        try:
126
            while len(self._devices):
127
                device = self._devices.pop()
128
                device.destroy()
122
            while len(self._images):
123
                image = self._images.pop()
124
                image.destroy()
129 125
        finally:
130 126
            # Make sure those are executed even if one of the device.destroy
131 127
            # methods throws exeptions.
......
176 172
        self.out.success('done')
177 173
        return "/dev/mapper/%s" % snapshot
178 174

  
179
    def get_device(self, media):
180
        """Returns a newly created DiskDevice instance."""
175
    def get_image(self, media):
176
        """Returns a newly created ImageCreator instance."""
181 177

  
182
        new_device = DiskDevice(media, self.out)
183
        self._devices.append(new_device)
184
        new_device.enable()
185
        return new_device
178
        image = Image(media, self.out)
179
        self._images.append(image)
180
        image.enable()
181
        return image
186 182

  
187
    def destroy_device(self, device):
188
        """Destroys a DiskDevice instance previously created by
189
        get_device method.
183
    def destroy_image(self, image):
184
        """Destroys an ImageCreator instance previously created by
185
        get_image_creator method.
190 186
        """
191
        self._devices.remove(device)
192
        device.destroy()
193

  
194

  
195
class DiskDevice(object):
196
    """This class represents a block device hosting an Operating System
197
    as created by the device-mapper.
198
    """
199

  
200
    def __init__(self, device, output, bootable=True, meta={}):
201
        """Create a new DiskDevice."""
202

  
203
        self.real_device = device
204
        self.out = output
205
        self.bootable = bootable
206
        self.meta = meta
207
        self.progress_bar = None
208
        self.guestfs_device = None
209
        self.size = 0
210

  
211
        self.g = guestfs.GuestFS()
212
        self.g.add_drive_opts(self.real_device, readonly=0, format="raw")
213

  
214
        # Before version 1.17.14 the recovery process, which is a fork of the
215
        # original process that called libguestfs, did not close its inherited
216
        # file descriptors. This can cause problems especially if the parent
217
        # process has opened pipes. Since the recovery process is an optional
218
        # feature of libguestfs, it's better to disable it.
219
        self.g.set_recovery_proc(0)
220
        version = self.g.version()
221
        if version['major'] > 1 or \
222
            (version['major'] == 1 and (version['minor'] >= 18 or
223
                                        (version['minor'] == 17 and
224
                                         version['release'] >= 14))):
225
            self.g.set_recovery_proc(1)
226
            self.out.output("Enabling recovery proc")
227

  
228
        #self.g.set_trace(1)
229
        #self.g.set_verbose(1)
230

  
231
        self.guestfs_enabled = False
232

  
233
    def enable(self):
234
        """Enable a newly created DiskDevice"""
235

  
236
        self.out.output('Launching helper VM (may take a while) ...', False)
237
        # self.progressbar = self.out.Progress(100, "Launching helper VM",
238
        #                                     "percent")
239
        # eh = self.g.set_event_callback(self.progress_callback,
240
        #                               guestfs.EVENT_PROGRESS)
241
        self.g.launch()
242
        self.guestfs_enabled = True
243
        # self.g.delete_event_callback(eh)
244
        # self.progressbar.success('done')
245
        # self.progressbar = None
246
        self.out.success('done')
247

  
248
        self.out.output('Inspecting Operating System ...', False)
249
        roots = self.g.inspect_os()
250
        if len(roots) == 0:
251
            raise FatalError("No operating system found")
252
        if len(roots) > 1:
253
            raise FatalError("Multiple operating systems found."
254
                             "We only support images with one OS.")
255
        self.root = roots[0]
256
        self.guestfs_device = self.g.part_to_dev(self.root)
257
        self.size = self.g.blockdev_getsize64(self.guestfs_device)
258
        self.meta['PARTITION_TABLE'] = \
259
            self.g.part_get_parttype(self.guestfs_device)
260

  
261
        self.ostype = self.g.inspect_get_type(self.root)
262
        self.distro = self.g.inspect_get_distro(self.root)
263
        self.out.success('found a(n) %s system' % self.distro)
264

  
265
    def destroy(self):
266
        """Destroy this DiskDevice instance."""
267

  
268
        # In new guestfs versions, there is a handy shutdown method for this
269
        try:
270
            if self.guestfs_enabled:
271
                self.g.umount_all()
272
                self.g.sync()
273
        finally:
274
            # Close the guestfs handler if open
275
            self.g.close()
276

  
277
#    def progress_callback(self, ev, eh, buf, array):
278
#        position = array[2]
279
#        total = array[3]
280
#
281
#        self.progressbar.goto((position * 100) // total)
282

  
283
    def mount(self, readonly=False):
284
        """Mount all disk partitions in a correct order."""
285

  
286
        mount = self.g.mount_ro if readonly else self.g.mount
287
        msg = " read-only" if readonly else ""
288
        self.out.output("Mounting the media%s ..." % msg, False)
289
        mps = self.g.inspect_get_mountpoints(self.root)
290

  
291
        # Sort the keys to mount the fs in a correct order.
292
        # / should be mounted befor /boot, etc
293
        def compare(a, b):
294
            if len(a[0]) > len(b[0]):
295
                return 1
296
            elif len(a[0]) == len(b[0]):
297
                return 0
298
            else:
299
                return -1
300
        mps.sort(compare)
301
        for mp, dev in mps:
302
            try:
303
                mount(dev, mp)
304
            except RuntimeError as msg:
305
                self.out.warn("%s (ignored)" % msg)
306
        self.out.success("done")
307

  
308
    def umount(self):
309
        """Umount all mounted filesystems."""
310
        self.g.umount_all()
311

  
312
    def _last_partition(self):
313
        if self.meta['PARTITION_TABLE'] not in 'msdos' 'gpt':
314
            msg = "Unsupported partition table: %s. Only msdos and gpt " \
315
                "partition tables are supported" % self.meta['PARTITION_TABLE']
316
            raise FatalError(msg)
317

  
318
        is_extended = lambda p: \
319
            self.g.part_get_mbr_id(self.guestfs_device, p['part_num']) \
320
            in (0x5, 0xf)
321
        is_logical = lambda p: \
322
            self.meta['PARTITION_TABLE'] == 'msdos' and p['part_num'] > 4
323

  
324
        partitions = self.g.part_list(self.guestfs_device)
325
        last_partition = partitions[-1]
326

  
327
        if is_logical(last_partition):
328
            # The disk contains extended and logical partitions....
329
            extended = filter(is_extended, partitions)[0]
330
            last_primary = [p for p in partitions if p['part_num'] <= 4][-1]
331

  
332
            # check if extended is the last primary partition
333
            if last_primary['part_num'] > extended['part_num']:
334
                last_partition = last_primary
335

  
336
        return last_partition
337

  
338
    def shrink(self):
339
        """Shrink the disk.
340

  
341
        This is accomplished by shrinking the last filesystem in the
342
        disk and then updating the partition table. The new disk size
343
        (in bytes) is returned.
344

  
345
        ATTENTION: make sure unmount is called before shrink
346
        """
347
        get_fstype = lambda p: \
348
            self.g.vfs_type("%s%d" % (self.guestfs_device, p['part_num']))
349
        is_logical = lambda p: \
350
            self.meta['PARTITION_TABLE'] == 'msdos' and p['part_num'] > 4
351
        is_extended = lambda p: \
352
            self.meta['PARTITION_TABLE'] == 'msdos' and \
353
            self.g.part_get_mbr_id(self.guestfs_device, p['part_num']) \
354
            in (0x5, 0xf)
355

  
356
        part_add = lambda ptype, start, stop: \
357
            self.g.part_add(self.guestfs_device, ptype, start, stop)
358
        part_del = lambda p: self.g.part_del(self.guestfs_device, p)
359
        part_get_id = lambda p: self.g.part_get_mbr_id(self.guestfs_device, p)
360
        part_set_id = lambda p, id: \
361
            self.g.part_set_mbr_id(self.guestfs_device, p, id)
362
        part_get_bootable = lambda p: \
363
            self.g.part_get_bootable(self.guestfs_device, p)
364
        part_set_bootable = lambda p, bootable: \
365
            self.g.part_set_bootable(self.guestfs_device, p, bootable)
366

  
367
        MB = 2 ** 20
368

  
369
        self.out.output("Shrinking image (this may take a while) ...", False)
370

  
371
        sector_size = self.g.blockdev_getss(self.guestfs_device)
372

  
373
        last_part = None
374
        fstype = None
375
        while True:
376
            last_part = self._last_partition()
377
            fstype = get_fstype(last_part)
378

  
379
            if fstype == 'swap':
380
                self.meta['SWAP'] = "%d:%s" % \
381
                    (last_part['part_num'],
382
                     (last_part['part_size'] + MB - 1) // MB)
383
                part_del(last_part['part_num'])
384
                continue
385
            elif is_extended(last_part):
386
                part_del(last_part['part_num'])
387
                continue
388

  
389
            # Most disk manipulation programs leave 2048 sectors after the last
390
            # partition
391
            new_size = last_part['part_end'] + 1 + 2048 * sector_size
392
            self.size = min(self.size, new_size)
393
            break
394

  
395
        if not re.match("ext[234]", fstype):
396
            self.out.warn("Don't know how to resize %s partitions." % fstype)
397
            return self.size
398

  
399
        part_dev = "%s%d" % (self.guestfs_device, last_part['part_num'])
400
        self.g.e2fsck_f(part_dev)
401
        self.g.resize2fs_M(part_dev)
402

  
403
        out = self.g.tune2fs_l(part_dev)
404
        block_size = int(filter(lambda x: x[0] == 'Block size', out)[0][1])
405
        block_cnt = int(filter(lambda x: x[0] == 'Block count', out)[0][1])
406

  
407
        start = last_part['part_start'] / sector_size
408
        end = start + (block_size * block_cnt) / sector_size - 1
409

  
410
        if is_logical(last_part):
411
            partitions = self.g.part_list(self.guestfs_device)
412

  
413
            logical = []  # logical partitions
414
            for partition in partitions:
415
                if partition['part_num'] < 4:
416
                    continue
417
                logical.append({
418
                    'num': partition['part_num'],
419
                    'start': partition['part_start'] / sector_size,
420
                    'end': partition['part_end'] / sector_size,
421
                    'id': part_get_id(partition['part_num']),
422
                    'bootable': part_get_bootable(partition['part_num'])
423
                })
424

  
425
            logical[-1]['end'] = end  # new end after resize
426

  
427
            # Recreate the extended partition
428
            extended = filter(is_extended, partitions)[0]
429
            part_del(extended['part_num'])
430
            part_add('e', extended['part_start'] / sector_size, end)
431

  
432
            # Create all the logical partitions back
433
            for l in logical:
434
                part_add('l', l['start'], l['end'])
435
                part_set_id(l['num'], l['id'])
436
                part_set_bootable(l['num'], l['bootable'])
437
        else:
438
            # Recreate the last partition
439
            if self.meta['PARTITION_TABLE'] == 'msdos':
440
                last_part['id'] = part_get_id(last_part['part_num'])
441

  
442
            last_part['bootable'] = part_get_bootable(last_part['part_num'])
443
            part_del(last_part['part_num'])
444
            part_add('p', start, end)
445
            part_set_bootable(last_part['part_num'], last_part['bootable'])
446

  
447
            if self.meta['PARTITION_TABLE'] == 'msdos':
448
                part_set_id(last_part['part_num'], last_part['id'])
449

  
450
        new_size = (end + 1) * sector_size
451

  
452
        assert (new_size <= self.size)
453

  
454
        if self.meta['PARTITION_TABLE'] == 'gpt':
455
            ptable = GPTPartitionTable(self.real_device)
456
            self.size = ptable.shrink(new_size, self.size)
457
        else:
458
            self.size = min(new_size + 2048 * sector_size, self.size)
459

  
460
        self.out.success("new size is %dMB" % ((self.size + MB - 1) // MB))
461

  
462
        return self.size
463

  
464
    def dump(self, outfile):
465
        """Dumps the content of device into a file.
466

  
467
        This method will only dump the actual payload, found by reading the
468
        partition table. Empty space in the end of the device will be ignored.
469
        """
470
        MB = 2 ** 20
471
        blocksize = 4 * MB  # 4MB
472
        size = self.size
473
        progr_size = (size + MB - 1) // MB  # in MB
474
        progressbar = self.out.Progress(progr_size, "Dumping image file", 'mb')
475

  
476
        with open(self.real_device, 'r') as src:
477
            with open(outfile, "w") as dst:
478
                left = size
479
                offset = 0
480
                progressbar.next()
481
                while left > 0:
482
                    length = min(left, blocksize)
483
                    sent = sendfile(dst.fileno(), src.fileno(), offset, length)
484

  
485
                    # Workaround for python-sendfile API change. In
486
                    # python-sendfile 1.2.x (py-sendfile) the returning value
487
                    # of sendfile is a tuple, where in version 2.x (pysendfile)
488
                    # it is just a sigle integer.
489
                    if isinstance(sent, tuple):
490
                        sent = sent[1]
491

  
492
                    offset += sent
493
                    left -= sent
494
                    progressbar.goto((size - left) // MB)
495
        progressbar.success('image file %s was successfully created' % outfile)
187
        self._images.remove(image)
188
        image.destroy()
496 189

  
497 190
# vim: set sta sts=4 shiftwidth=4 sw=4 et ai :
b/image_creator/image.py
1
# Copyright 2013 GRNET S.A. All rights reserved.
2
#
3
# Redistribution and use in source and binary forms, with or
4
# without modification, are permitted provided that the following
5
# conditions are met:
6
#
7
#   1. Redistributions of source code must retain the above
8
#      copyright notice, this list of conditions and the following
9
#      disclaimer.
10
#
11
#   2. Redistributions in binary form must reproduce the above
12
#      copyright notice, this list of conditions and the following
13
#      disclaimer in the documentation and/or other materials
14
#      provided with the distribution.
15
#
16
# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
17
# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
18
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
19
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
20
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
21
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
22
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
23
# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
24
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
26
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
27
# POSSIBILITY OF SUCH DAMAGE.
28
#
29
# The views and conclusions contained in the software and
30
# documentation are those of the authors and should not be
31
# interpreted as representing official policies, either expressed
32
# or implied, of GRNET S.A.
33

  
34
from image_creator.util import FatalError
35
from image_creator.gpt import GPTPartitionTable
36
from image_creator.os_type import os_cls
37

  
38
import re
39
import guestfs
40
from sendfile import sendfile
41

  
42

  
43
class Image(object):
44
    """The instances of this class can create images out of block devices."""
45

  
46
    def __init__(self, device, output, bootable=True, meta={}):
47
        """Create a new ImageCreator."""
48

  
49
        self.device = device
50
        self.out = output
51
        self.bootable = bootable
52
        self.meta = meta
53
        self.progress_bar = None
54
        self.guestfs_device = None
55
        self.size = 0
56
        self.mounted = False
57

  
58
        self.g = guestfs.GuestFS()
59
        self.g.add_drive_opts(self.device, readonly=0, format="raw")
60

  
61
        # Before version 1.17.14 the recovery process, which is a fork of the
62
        # original process that called libguestfs, did not close its inherited
63
        # file descriptors. This can cause problems especially if the parent
64
        # process has opened pipes. Since the recovery process is an optional
65
        # feature of libguestfs, it's better to disable it.
66
        self.g.set_recovery_proc(0)
67
        version = self.g.version()
68
        if version['major'] > 1 or \
69
            (version['major'] == 1 and (version['minor'] >= 18 or
70
                                        (version['minor'] == 17 and
71
                                         version['release'] >= 14))):
72
            self.g.set_recovery_proc(1)
73
            self.out.output("Enabling recovery proc")
74

  
75
        #self.g.set_trace(1)
76
        #self.g.set_verbose(1)
77

  
78
        self.guestfs_enabled = False
79

  
80
    def enable(self):
81
        """Enable a newly created ImageCreator"""
82

  
83
        self.out.output('Launching helper VM (may take a while) ...', False)
84
        # self.progressbar = self.out.Progress(100, "Launching helper VM",
85
        #                                     "percent")
86
        # eh = self.g.set_event_callback(self.progress_callback,
87
        #                               guestfs.EVENT_PROGRESS)
88
        self.g.launch()
89
        self.guestfs_enabled = True
90
        # self.g.delete_event_callback(eh)
91
        # self.progressbar.success('done')
92
        # self.progressbar = None
93
        self.out.success('done')
94

  
95
        self.out.output('Inspecting Operating System ...', False)
96
        roots = self.g.inspect_os()
97
        if len(roots) == 0:
98
            raise FatalError("No operating system found")
99
        if len(roots) > 1:
100
            raise FatalError("Multiple operating systems found."
101
                             "We only support images with one OS.")
102
        self.root = roots[0]
103
        self.guestfs_device = self.g.part_to_dev(self.root)
104
        self.size = self.g.blockdev_getsize64(self.guestfs_device)
105
        self.meta['PARTITION_TABLE'] = \
106
            self.g.part_get_parttype(self.guestfs_device)
107

  
108
        self.ostype = self.g.inspect_get_type(self.root)
109
        self.distro = self.g.inspect_get_distro(self.root)
110
        self.out.success('found a(n) %s system' % self.distro)
111

  
112
    def _get_os(self):
113
        if hasattr(self, "_os"):
114
            return self._os
115

  
116
        if not self.guestfs_enabled:
117
            self.enable()
118

  
119
        if not self.mounted:
120
            do_unmount = True
121
            self.mount(readonly=True)
122
        else:
123
            do_unmount = False
124

  
125
        try:
126
            cls = os_cls(self.distro, self.ostype)
127
            self._os = cls(self.root, self.g, self.out)
128

  
129
        finally:
130
            if do_unmount:
131
                self.umount()
132

  
133
        return self._os
134

  
135
    os = property(_get_os)
136

  
137
    def destroy(self):
138
        """Destroy this ImageCreator instance."""
139

  
140
        # In new guestfs versions, there is a handy shutdown method for this
141
        try:
142
            if self.guestfs_enabled:
143
                self.g.umount_all()
144
                self.g.sync()
145
        finally:
146
            # Close the guestfs handler if open
147
            self.g.close()
148

  
149
#    def progress_callback(self, ev, eh, buf, array):
150
#        position = array[2]
151
#        total = array[3]
152
#
153
#        self.progressbar.goto((position * 100) // total)
154

  
155
    def mount(self, readonly=False):
156
        """Mount all disk partitions in a correct order."""
157

  
158
        mount = self.g.mount_ro if readonly else self.g.mount
159
        msg = " read-only" if readonly else ""
160
        self.out.output("Mounting the media%s ..." % msg, False)
161
        mps = self.g.inspect_get_mountpoints(self.root)
162

  
163
        # Sort the keys to mount the fs in a correct order.
164
        # / should be mounted befor /boot, etc
165
        def compare(a, b):
166
            if len(a[0]) > len(b[0]):
167
                return 1
168
            elif len(a[0]) == len(b[0]):
169
                return 0
170
            else:
171
                return -1
172
        mps.sort(compare)
173
        for mp, dev in mps:
174
            try:
175
                mount(dev, mp)
176
            except RuntimeError as msg:
177
                self.out.warn("%s (ignored)" % msg)
178

  
179
        self.mounted = True
180
        self.out.success("done")
181

  
182
    def umount(self):
183
        """Umount all mounted filesystems."""
184
        self.g.umount_all()
185
        self.mounted = False
186

  
187
    def _last_partition(self):
188
        if self.meta['PARTITION_TABLE'] not in 'msdos' 'gpt':
189
            msg = "Unsupported partition table: %s. Only msdos and gpt " \
190
                "partition tables are supported" % self.meta['PARTITION_TABLE']
191
            raise FatalError(msg)
192

  
193
        is_extended = lambda p: \
194
            self.g.part_get_mbr_id(self.guestfs_device, p['part_num']) \
195
            in (0x5, 0xf)
196
        is_logical = lambda p: \
197
            self.meta['PARTITION_TABLE'] == 'msdos' and p['part_num'] > 4
198

  
199
        partitions = self.g.part_list(self.guestfs_device)
200
        last_partition = partitions[-1]
201

  
202
        if is_logical(last_partition):
203
            # The disk contains extended and logical partitions....
204
            extended = filter(is_extended, partitions)[0]
205
            last_primary = [p for p in partitions if p['part_num'] <= 4][-1]
206

  
207
            # check if extended is the last primary partition
208
            if last_primary['part_num'] > extended['part_num']:
209
                last_partition = last_primary
210

  
211
        return last_partition
212

  
213
    def shrink(self):
214
        """Shrink the disk.
215

  
216
        This is accomplished by shrinking the last filesystem in the
217
        disk and then updating the partition table. The new disk size
218
        (in bytes) is returned.
219

  
220
        ATTENTION: make sure unmount is called before shrink
221
        """
222
        get_fstype = lambda p: \
223
            self.g.vfs_type("%s%d" % (self.guestfs_device, p['part_num']))
224
        is_logical = lambda p: \
225
            self.meta['PARTITION_TABLE'] == 'msdos' and p['part_num'] > 4
226
        is_extended = lambda p: \
227
            self.meta['PARTITION_TABLE'] == 'msdos' and \
228
            self.g.part_get_mbr_id(self.guestfs_device, p['part_num']) \
229
            in (0x5, 0xf)
230

  
231
        part_add = lambda ptype, start, stop: \
232
            self.g.part_add(self.guestfs_device, ptype, start, stop)
233
        part_del = lambda p: self.g.part_del(self.guestfs_device, p)
234
        part_get_id = lambda p: self.g.part_get_mbr_id(self.guestfs_device, p)
235
        part_set_id = lambda p, id: \
236
            self.g.part_set_mbr_id(self.guestfs_device, p, id)
237
        part_get_bootable = lambda p: \
238
            self.g.part_get_bootable(self.guestfs_device, p)
239
        part_set_bootable = lambda p, bootable: \
240
            self.g.part_set_bootable(self.guestfs_device, p, bootable)
241

  
242
        MB = 2 ** 20
243

  
244
        self.out.output("Shrinking image (this may take a while) ...", False)
245

  
246
        sector_size = self.g.blockdev_getss(self.guestfs_device)
247

  
248
        last_part = None
249
        fstype = None
250
        while True:
251
            last_part = self._last_partition()
252
            fstype = get_fstype(last_part)
253

  
254
            if fstype == 'swap':
255
                self.meta['SWAP'] = "%d:%s" % \
256
                    (last_part['part_num'],
257
                     (last_part['part_size'] + MB - 1) // MB)
258
                part_del(last_part['part_num'])
259
                continue
260
            elif is_extended(last_part):
261
                part_del(last_part['part_num'])
262
                continue
263

  
264
            # Most disk manipulation programs leave 2048 sectors after the last
265
            # partition
266
            new_size = last_part['part_end'] + 1 + 2048 * sector_size
267
            self.size = min(self.size, new_size)
268
            break
269

  
270
        if not re.match("ext[234]", fstype):
271
            self.out.warn("Don't know how to resize %s partitions." % fstype)
272
            return self.size
273

  
274
        part_dev = "%s%d" % (self.guestfs_device, last_part['part_num'])
275
        self.g.e2fsck_f(part_dev)
276
        self.g.resize2fs_M(part_dev)
277

  
278
        out = self.g.tune2fs_l(part_dev)
279
        block_size = int(filter(lambda x: x[0] == 'Block size', out)[0][1])
280
        block_cnt = int(filter(lambda x: x[0] == 'Block count', out)[0][1])
281

  
282
        start = last_part['part_start'] / sector_size
283
        end = start + (block_size * block_cnt) / sector_size - 1
284

  
285
        if is_logical(last_part):
286
            partitions = self.g.part_list(self.guestfs_device)
287

  
288
            logical = []  # logical partitions
289
            for partition in partitions:
290
                if partition['part_num'] < 4:
291
                    continue
292
                logical.append({
293
                    'num': partition['part_num'],
294
                    'start': partition['part_start'] / sector_size,
295
                    'end': partition['part_end'] / sector_size,
296
                    'id': part_get_id(partition['part_num']),
297
                    'bootable': part_get_bootable(partition['part_num'])
298
                })
299

  
300
            logical[-1]['end'] = end  # new end after resize
301

  
302
            # Recreate the extended partition
303
            extended = filter(is_extended, partitions)[0]
304
            part_del(extended['part_num'])
305
            part_add('e', extended['part_start'] / sector_size, end)
306

  
307
            # Create all the logical partitions back
308
            for l in logical:
309
                part_add('l', l['start'], l['end'])
310
                part_set_id(l['num'], l['id'])
311
                part_set_bootable(l['num'], l['bootable'])
312
        else:
313
            # Recreate the last partition
314
            if self.meta['PARTITION_TABLE'] == 'msdos':
315
                last_part['id'] = part_get_id(last_part['part_num'])
316

  
317
            last_part['bootable'] = part_get_bootable(last_part['part_num'])
318
            part_del(last_part['part_num'])
319
            part_add('p', start, end)
320
            part_set_bootable(last_part['part_num'], last_part['bootable'])
321

  
322
            if self.meta['PARTITION_TABLE'] == 'msdos':
323
                part_set_id(last_part['part_num'], last_part['id'])
324

  
325
        new_size = (end + 1) * sector_size
326

  
327
        assert (new_size <= self.size)
328

  
329
        if self.meta['PARTITION_TABLE'] == 'gpt':
330
            ptable = GPTPartitionTable(self.device)
331
            self.size = ptable.shrink(new_size, self.size)
332
        else:
333
            self.size = min(new_size + 2048 * sector_size, self.size)
334

  
335
        self.out.success("new size is %dMB" % ((self.size + MB - 1) // MB))
336

  
337
        return self.size
338

  
339
    def dump(self, outfile):
340
        """Dumps the content of device into a file.
341

  
342
        This method will only dump the actual payload, found by reading the
343
        partition table. Empty space in the end of the device will be ignored.
344
        """
345
        MB = 2 ** 20
346
        blocksize = 4 * MB  # 4MB
347
        size = self.size
348
        progr_size = (size + MB - 1) // MB  # in MB
349
        progressbar = self.out.Progress(progr_size, "Dumping image file", 'mb')
350

  
351
        with open(self.device, 'r') as src:
352
            with open(outfile, "w") as dst:
353
                left = size
354
                offset = 0
355
                progressbar.next()
356
                while left > 0:
357
                    length = min(left, blocksize)
358
                    sent = sendfile(dst.fileno(), src.fileno(), offset, length)
359

  
360
                    # Workaround for python-sendfile API change. In
361
                    # python-sendfile 1.2.x (py-sendfile) the returning value
362
                    # of sendfile is a tuple, where in version 2.x (pysendfile)
363
                    # it is just a sigle integer.
364
                    if isinstance(sent, tuple):
365
                        sent = sent[1]
366

  
367
                    offset += sent
368
                    left -= sent
369
                    progressbar.goto((size - left) // MB)
370
        progressbar.success('image file %s was successfully created' % outfile)
371

  
372
# vim: set sta sts=4 shiftwidth=4 sw=4 et ai :
b/image_creator/main.py
38 38
from image_creator.util import FatalError, MD5
39 39
from image_creator.output.cli import SilentOutput, SimpleOutput, \
40 40
    OutputWthProgress
41
from image_creator.os_type import os_cls
42 41
from image_creator.kamaki_wrapper import Kamaki, ClientError
43 42
import sys
44 43
import os
......
205 204
    try:
206 205
        snapshot = disk.snapshot()
207 206

  
208
        dev = disk.get_device(snapshot)
207
        image = disk.get_image(snapshot)
209 208

  
210 209
        # If no customization is to be applied, the image should be mounted ro
211
        readonly = (not (options.sysprep or options.shrink) or
212
                    options.print_sysprep)
213
        dev.mount(readonly)
214

  
215
        cls = os_cls(dev.distro, dev.ostype)
216
        image_os = cls(dev.root, dev.g, out)
217
        out.output()
218

  
219
        for sysprep in options.disabled_syspreps:
220
            image_os.disable_sysprep(image_os.get_sysprep_by_name(sysprep))
210
        ro = (not (options.sysprep or options.shrink) or options.print_sysprep)
211
        image.mount(ro)
212
        try:
213
            for sysprep in options.disabled_syspreps:
214
                image.os.disable_sysprep(image.os.get_sysprep_by_name(sysprep))
221 215

  
222
        for sysprep in options.enabled_syspreps:
223
            image_os.enable_sysprep(image_os.get_sysprep_by_name(sysprep))
216
            for sysprep in options.enabled_syspreps:
217
                image.os.enable_sysprep(image.os.get_sysprep_by_name(sysprep))
224 218

  
225
        if options.print_sysprep:
226
            image_os.print_syspreps()
227
            out.output()
219
            if options.print_sysprep:
220
                image.os.print_syspreps()
221
                out.output()
228 222

  
229
        if options.outfile is None and not options.upload:
230
            return 0
223
            if options.outfile is None and not options.upload:
224
                return 0
231 225

  
232
        if options.sysprep:
233
            image_os.do_sysprep()
226
            if options.sysprep:
227
                image.os.do_sysprep()
234 228

  
235
        metadata = image_os.meta
236
        dev.umount()
229
            metadata = image.os.meta
230
        finally:
231
            image.umount()
237 232

  
238
        size = options.shrink and dev.shrink() or dev.size
239
        metadata.update(dev.meta)
233
        size = options.shrink and image.shrink() or image.size
234
        metadata.update(image.meta)
240 235

  
241 236
        # Add command line metadata to the collected ones...
242 237
        metadata.update(options.metadata)
243 238

  
244 239
        md5 = MD5(out)
245
        checksum = md5.compute(snapshot, size)
240
        checksum = md5.compute(image.device, size)
246 241

  
247 242
        metastring = '\n'.join(
248 243
            ['%s=%s' % (key, value) for (key, value) in metadata.items()])
249 244
        metastring += '\n'
250 245

  
251 246
        if options.outfile is not None:
252
            dev.dump(options.outfile)
247
            image.dump(options.outfile)
253 248

  
254 249
            out.output('Dumping metadata file ...', False)
255 250
            with open('%s.%s' % (options.outfile, 'meta'), 'w') as f:
......
262 257
                                     os.path.basename(options.outfile)))
263 258
            out.success('done')
264 259

  
265
        # Destroy the device. We only need the snapshot from now on
266
        disk.destroy_device(dev)
260
        # Destroy the image instance. We only need the snapshot from now on
261
        disk.destroy_image(image)
267 262

  
268 263
        out.output()
269 264
        try:

Also available in: Unified diff