From c16850ae1ce37e8fed14320c55233e272bed4e69 Mon Sep 17 00:00:00 2001 From: Nikos Skalkotos Date: Thu, 10 Jan 2013 18:57:16 +0200 Subject: [PATCH] Allow users to specify tmp dir for large files The cow file when creating images or the temporary image file when bundling the host system may be large. Allow the user to overwrite the directory under which those files get created. --- image_creator/bundle_volume.py | 10 ++++++---- image_creator/dialog_main.py | 16 ++++++++++++---- image_creator/disk.py | 30 +++++++++++++++++++++++++++--- image_creator/main.py | 20 ++++++++++++++------ image_creator/util.py | 6 ++++++ 5 files changed, 65 insertions(+), 17 deletions(-) diff --git a/image_creator/bundle_volume.py b/image_creator/bundle_volume.py index 450d50d..fc35352 100644 --- a/image_creator/bundle_volume.py +++ b/image_creator/bundle_volume.py @@ -42,6 +42,7 @@ from image_creator.rsync import Rsync from image_creator.util import get_command from image_creator.util import FatalError from image_creator.util import try_fail_repeat +from image_creator.util import free_space findfs = get_command('findfs') dd = get_command('dd') @@ -67,10 +68,11 @@ MKFS_OPTS = {'ext2': ['-F'], class BundleVolume(object): """This class can be used to create an image out of the running system""" - def __init__(self, out, meta): + def __init__(self, out, meta, tmp=None): """Create an instance of the BundleVolume class.""" self.out = out self.meta = meta + self.tmp = tmp self.out.output('Searching for root device ...', False) root = self._get_root_partition() @@ -279,6 +281,8 @@ class BundleVolume(object): def _to_exclude(self): excluded = ['/tmp', '/var/tmp'] + if self.tmp is not None: + excluded.append(self.tmp) local_filesystems = MKFS_OPTS.keys() + ['rootfs'] for entry in self._read_fstable('/proc/mounts'): if entry.fs in local_filesystems: @@ -430,9 +434,7 @@ class BundleVolume(object): # Check if the available space is enough to host the image dirname = os.path.dirname(image) self.out.output("Examining available space in %s ..." % dirname, False) - stat = os.statvfs(dirname) - available = stat.f_bavail * stat.f_frsize - if available <= size: + if free_space(dirname) <= size: raise FatalError('Not enough space in %s to host the image' % dirname) self.out.success("sufficient") diff --git a/image_creator/dialog_main.py b/image_creator/dialog_main.py index a32a977..21fa873 100644 --- a/image_creator/dialog_main.py +++ b/image_creator/dialog_main.py @@ -55,13 +55,13 @@ from image_creator.dialog_util import SMALL_WIDTH, WIDTH, confirm_exit, \ Reset, update_background_title -def image_creator(d, media, out): +def image_creator(d, media, out, tmp): d.setBackgroundTitle('snf-image-creator') gauge = GaugeOutput(d, "Initialization", "Initializing...") out.add(gauge) - disk = Disk(media, out) + disk = Disk(media, out, tmp) def signal_handler(signum, frame): gauge.cleanup() @@ -183,6 +183,9 @@ def main(): parser.add_option("-l", "--logfile", type="string", dest="logfile", default=None, help="log all messages to FILE", metavar="FILE") + parser.add_option("--tmpdir", type="string", dest="tmp", default=None, + help="create large temporary image files under DIR", + metavar="DIR") options, args = parser.parse_args(sys.argv[1:]) @@ -196,7 +199,9 @@ def main(): raise FatalError("You must run %s as root" % parser.get_prog_name()) - media = select_file(d, args[0] if len(args) == 1 else None) + if options.tmp is not None and not os.path.isdir(options.tmp): + raise FatalError("The directory `%s' specified with --tmpdir is " + "not valid" % options.tmp) logfile = None if options.logfile is not None: @@ -206,6 +211,9 @@ def main(): raise FatalError( "Unable to open logfile `%s' for writing. Reason: %s" % (options.logfile, e.strerror)) + + media = select_file(d, args[0] if len(args) == 1 else None) + try: log = SimpleOutput(False, logfile) if logfile is not None \ else Output() @@ -214,7 +222,7 @@ def main(): out = CompositeOutput([log]) out.output("Starting %s v%s..." % (parser.get_prog_name(), version)) - ret = image_creator(d, media, out) + ret = image_creator(d, media, out, options.tmp) sys.exit(ret) except Reset: log.output("Resetting everything...") diff --git a/image_creator/disk.py b/image_creator/disk.py index dc7cbd8..4209462 100644 --- a/image_creator/disk.py +++ b/image_creator/disk.py @@ -34,6 +34,7 @@ from image_creator.util import get_command from image_creator.util import FatalError from image_creator.util import try_fail_repeat +from image_creator.util import free_space from image_creator.gpt import GPTPartitionTable from image_creator.bundle_volume import BundleVolume @@ -53,6 +54,9 @@ losetup = get_command('losetup') blockdev = get_command('blockdev') +TMP_CANDIDATES = ['/var/tmp', os.path.expanduser('~'), '/mnt'] + + class Disk(object): """This class represents a hard disk hosting an Operating System @@ -61,7 +65,7 @@ class Disk(object): the Linux kernel. """ - def __init__(self, source, output): + def __init__(self, source, output, tmp=None): """Create a new Disk instance out of a source media. The source media can be an image file, a block device or a directory. """ @@ -70,6 +74,26 @@ class Disk(object): self.source = source self.out = output self.meta = {} + self.tmp = tempfile.mkdtemp(prefix='.snf_image_creator.', + dir=self._get_tmp_dir(tmp)) + + self._add_cleanup(os.removedirs, self.tmp) + + def _get_tmp_dir(self, default=None): + if default is not None: + return default + + space = map(free_space, TMP_CANDIDATES) + + max_idx = 0 + max_val = space[0] + for i, val in zip(range(len(space)), space): + if val > max_val: + max_val = val + max_idx = i + + # Return the candidate path with more available space + return TMP_CANDIDATES[max_idx] def _add_cleanup(self, job, *args): self._cleanup_jobs.append((job, args)) @@ -83,7 +107,7 @@ class Disk(object): def _dir_to_disk(self): if self.source == '/': bundle = BundleVolume(self.out, self.meta) - image = '/var/tmp/%s.diskdump' % uuid.uuid4().hex + image = '%s/%s.diskdump' % (self.tmp, uuid.uuid4().hex) def check_unlink(path): if os.path.exists(path): @@ -132,7 +156,7 @@ class Disk(object): # Take a snapshot and return it to the user self.out.output("Snapshotting media source...", False) size = blockdev('--getsz', sourcedev) - cowfd, cow = tempfile.mkstemp() + cowfd, cow = tempfile.mkstemp(dir=self.tmp) os.close(cowfd) self._add_cleanup(os.unlink, cow) # Create cow sparse file diff --git a/image_creator/main.py b/image_creator/main.py index 3161bb5..39144bb 100644 --- a/image_creator/main.py +++ b/image_creator/main.py @@ -79,7 +79,7 @@ def parse_options(input_args): help="overwrite output files if they exist") parser.add_option("-s", "--silent", dest="silent", default=False, - help="silent mode, only output errors", + help="output only errors", action="store_true") parser.add_option("-u", "--upload", dest="upload", type="string", @@ -93,15 +93,15 @@ def parse_options(input_args): metavar="IMAGENAME") parser.add_option("-a", "--account", dest="account", type="string", - default=account, help="Use this ACCOUNT when " + default=account, help="use this ACCOUNT when " "uploading/registering images [Default: %s]" % account) parser.add_option("-m", "--metadata", dest="metadata", default=[], - help="Add custom KEY=VALUE metadata to the image", + help="add custom KEY=VALUE metadata to the image", action="append", metavar="KEY=VALUE") parser.add_option("-t", "--token", dest="token", type="string", - default=token, help="Use this token when " + default=token, help="use this token when " "uploading/registering images [Default: %s]" % token) parser.add_option("--print-sysprep", dest="print_sysprep", default=False, @@ -118,12 +118,16 @@ def parse_options(input_args): metavar="SYSPREP") parser.add_option("--no-sysprep", dest="sysprep", default=True, - help="don't perform system preparation", + help="don't perform any system preparation operation", action="store_false") parser.add_option("--no-shrink", dest="shrink", default=True, help="don't shrink any partition", action="store_false") + parser.add_option("--tmpdir", dest="tmp", type="string", default=None, + help="create large temporary image files under DIR", + metavar="DIR") + options, args = parser.parse_args(input_args) if len(args) != 1: @@ -145,6 +149,10 @@ def parse_options(input_args): raise FatalError("Image uploading cannot be performed. No ~okeanos " "token is specified. User -t to set a token.") + if options.tmp is not None and not os.path.isdir(options.tmp): + raise FatalError("The directory `%s' specified with --tmpdir is not " + "valid." % options.tmp) + meta = {} for m in options.metadata: try: @@ -187,7 +195,7 @@ def image_creator(): raise FatalError("Output file %s exists " "(use --force to overwrite it)." % filename) - disk = Disk(options.source, out) + disk = Disk(options.source, out, options.tmp) def signal_handler(signum, frame): disk.cleanup() diff --git a/image_creator/util.py b/image_creator/util.py index 1cd97ff..8730143 100644 --- a/image_creator/util.py +++ b/image_creator/util.py @@ -35,6 +35,7 @@ import sys import sh import hashlib import time +import os class FatalError(Exception): @@ -73,6 +74,11 @@ def try_fail_repeat(command, *args): raise FatalError("Command: `%s %s' failed" % (command, " ".join(args))) +def free_space(dirname): + stat = os.statvfs(dirname) + return stat.f_bavail * stat.f_frsize + + class MD5: def __init__(self, output): self.out = output -- 1.7.10.4