Revision c23bb217

b/Makefile.am
265 265
	lib/compat.py \
266 266
	lib/config.py \
267 267
	lib/constants.py \
268
	lib/container.py \
268 269
	lib/daemon.py \
269 270
	lib/errors.py \
270 271
	lib/hooksmaster.py \
......
288 289
	lib/serializer.py \
289 290
	lib/ssconf.py \
290 291
	lib/ssh.py \
291
	lib/storage.py \
292 292
	lib/uidpool.py \
293 293
	lib/vcluster.py \
294 294
	lib/network.py \
......
1153 1153
	test/py/ganeti.confd.client_unittest.py \
1154 1154
	test/py/ganeti.config_unittest.py \
1155 1155
	test/py/ganeti.constants_unittest.py \
1156
	test/py/ganeti.container_unittest.py \
1156 1157
	test/py/ganeti.daemon_unittest.py \
1157 1158
	test/py/ganeti.errors_unittest.py \
1158 1159
	test/py/ganeti.hooks_unittest.py \
......
1190 1191
	test/py/ganeti.server.rapi_unittest.py \
1191 1192
	test/py/ganeti.ssconf_unittest.py \
1192 1193
	test/py/ganeti.ssh_unittest.py \
1193
	test/py/ganeti.storage_unittest.py \
1194 1194
	test/py/ganeti.tools.burnin_unittest.py \
1195 1195
	test/py/ganeti.tools.ensure_dirs_unittest.py \
1196 1196
	test/py/ganeti.tools.node_daemon_setup_unittest.py \
b/lib/container.py
1
#
2
#
3

  
4
# Copyright (C) 2009, 2011, 2012 Google Inc.
5
#
6
# This program is free software; you can redistribute it and/or modify
7
# it under the terms of the GNU General Public License as published by
8
# the Free Software Foundation; either version 2 of the License, or
9
# (at your option) any later version.
10
#
11
# This program is distributed in the hope that it will be useful, but
12
# WITHOUT ANY WARRANTY; without even the implied warranty of
13
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
14
# General Public License for more details.
15
#
16
# You should have received a copy of the GNU General Public License
17
# along with this program; if not, write to the Free Software
18
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
19
# 02110-1301, USA.
20

  
21

  
22
"""Storage container abstraction.
23

  
24
"""
25

  
26
# pylint: disable=W0232,R0201
27

  
28
# W0232, since we use these as singletons rather than object holding
29
# data
30

  
31
# R0201, for the same reason
32

  
33
# TODO: FileStorage initialised with paths whereas the others not
34

  
35
import logging
36

  
37
from ganeti import errors
38
from ganeti import constants
39
from ganeti import utils
40

  
41

  
42
def _ParseSize(value):
43
  return int(round(float(value), 0))
44

  
45

  
46
class _Base:
47
  """Base class for storage abstraction.
48

  
49
  """
50
  def List(self, name, fields):
51
    """Returns a list of all entities within the storage unit.
52

  
53
    @type name: string or None
54
    @param name: Entity name or None for all
55
    @type fields: list
56
    @param fields: List with all requested result fields (order is preserved)
57

  
58
    """
59
    raise NotImplementedError()
60

  
61
  def Modify(self, name, changes): # pylint: disable=W0613
62
    """Modifies an entity within the storage unit.
63

  
64
    @type name: string
65
    @param name: Entity name
66
    @type changes: dict
67
    @param changes: New field values
68

  
69
    """
70
    # Don't raise an error if no changes are requested
71
    if changes:
72
      raise errors.ProgrammerError("Unable to modify the following"
73
                                   "fields: %r" % (changes.keys(), ))
74

  
75
  def Execute(self, name, op):
76
    """Executes an operation on an entity within the storage unit.
77

  
78
    @type name: string
79
    @param name: Entity name
80
    @type op: string
81
    @param op: Operation name
82

  
83
    """
84
    raise NotImplementedError()
85

  
86

  
87
class FileStorage(_Base): # pylint: disable=W0223
88
  """File storage unit.
89

  
90
  """
91
  def __init__(self, paths):
92
    """Initializes this class.
93

  
94
    @type paths: list
95
    @param paths: List of file storage paths
96

  
97
    """
98
    self._paths = paths
99

  
100
  def List(self, name, fields):
101
    """Returns a list of all entities within the storage unit.
102

  
103
    See L{_Base.List}.
104

  
105
    """
106
    rows = []
107

  
108
    if name is None:
109
      paths = self._paths
110
    else:
111
      paths = [name]
112

  
113
    for path in paths:
114
      rows.append(self._ListInner(path, fields))
115

  
116
    return rows
117

  
118
  @staticmethod
119
  def _ListInner(path, fields):
120
    """Gathers requested information from directory.
121

  
122
    @type path: string
123
    @param path: Path to directory
124
    @type fields: list
125
    @param fields: Requested fields
126

  
127
    """
128
    values = []
129

  
130
    # Pre-calculate information in case it's requested more than once
131
    if constants.SF_USED in fields:
132
      dirsize = utils.CalculateDirectorySize(path)
133
    else:
134
      dirsize = None
135

  
136
    if constants.SF_FREE in fields or constants.SF_SIZE in fields:
137
      fsstats = utils.GetFilesystemStats(path)
138
    else:
139
      fsstats = None
140

  
141
    # Make sure to update constants.VALID_STORAGE_FIELDS when changing fields.
142
    for field_name in fields:
143
      if field_name == constants.SF_NAME:
144
        values.append(path)
145

  
146
      elif field_name == constants.SF_USED:
147
        values.append(dirsize)
148

  
149
      elif field_name == constants.SF_FREE:
150
        values.append(fsstats[1])
151

  
152
      elif field_name == constants.SF_SIZE:
153
        values.append(fsstats[0])
154

  
155
      elif field_name == constants.SF_ALLOCATABLE:
156
        values.append(True)
157

  
158
      else:
159
        raise errors.StorageError("Unknown field: %r" % field_name)
160

  
161
    return values
162

  
163

  
164
class _LvmBase(_Base): # pylint: disable=W0223
165
  """Base class for LVM storage containers.
166

  
167
  @cvar LIST_FIELDS: list of tuples consisting of three elements: SF_*
168
      constants, lvm command output fields (list), and conversion
169
      function or static value (for static value, the lvm output field
170
      can be an empty list)
171

  
172
  """
173
  LIST_SEP = "|"
174
  LIST_COMMAND = None
175
  LIST_FIELDS = None
176

  
177
  def List(self, name, wanted_field_names):
178
    """Returns a list of all entities within the storage unit.
179

  
180
    See L{_Base.List}.
181

  
182
    """
183
    # Get needed LVM fields
184
    lvm_fields = self._GetLvmFields(self.LIST_FIELDS, wanted_field_names)
185

  
186
    # Build LVM command
187
    cmd_args = self._BuildListCommand(self.LIST_COMMAND, self.LIST_SEP,
188
                                      lvm_fields, name)
189

  
190
    # Run LVM command
191
    cmd_result = self._RunListCommand(cmd_args)
192

  
193
    # Split and rearrange LVM command output
194
    return self._BuildList(self._SplitList(cmd_result, self.LIST_SEP,
195
                                           len(lvm_fields)),
196
                           self.LIST_FIELDS,
197
                           wanted_field_names,
198
                           lvm_fields)
199

  
200
  @staticmethod
201
  def _GetLvmFields(fields_def, wanted_field_names):
202
    """Returns unique list of fields wanted from LVM command.
203

  
204
    @type fields_def: list
205
    @param fields_def: Field definitions
206
    @type wanted_field_names: list
207
    @param wanted_field_names: List of requested fields
208

  
209
    """
210
    field_to_idx = dict([(field_name, idx)
211
                         for (idx, (field_name, _, _)) in
212
                         enumerate(fields_def)])
213

  
214
    lvm_fields = []
215

  
216
    for field_name in wanted_field_names:
217
      try:
218
        idx = field_to_idx[field_name]
219
      except IndexError:
220
        raise errors.StorageError("Unknown field: %r" % field_name)
221

  
222
      (_, lvm_names, _) = fields_def[idx]
223

  
224
      lvm_fields.extend(lvm_names)
225

  
226
    return utils.UniqueSequence(lvm_fields)
227

  
228
  @classmethod
229
  def _BuildList(cls, cmd_result, fields_def, wanted_field_names, lvm_fields):
230
    """Builds the final result list.
231

  
232
    @type cmd_result: iterable
233
    @param cmd_result: Iterable of LVM command output (iterable of lists)
234
    @type fields_def: list
235
    @param fields_def: Field definitions
236
    @type wanted_field_names: list
237
    @param wanted_field_names: List of requested fields
238
    @type lvm_fields: list
239
    @param lvm_fields: LVM fields
240

  
241
    """
242
    lvm_name_to_idx = dict([(lvm_name, idx)
243
                           for (idx, lvm_name) in enumerate(lvm_fields)])
244
    field_to_idx = dict([(field_name, idx)
245
                         for (idx, (field_name, _, _)) in
246
                         enumerate(fields_def)])
247

  
248
    data = []
249
    for raw_data in cmd_result:
250
      row = []
251

  
252
      for field_name in wanted_field_names:
253
        (_, lvm_names, mapper) = fields_def[field_to_idx[field_name]]
254

  
255
        values = [raw_data[lvm_name_to_idx[i]] for i in lvm_names]
256

  
257
        if callable(mapper):
258
          # we got a function, call it with all the declared fields
259
          val = mapper(*values) # pylint: disable=W0142
260
        elif len(values) == 1:
261
          assert mapper is None, ("Invalid mapper value (neither callable"
262
                                  " nor None) for one-element fields")
263
          # we don't have a function, but we had a single field
264
          # declared, pass it unchanged
265
          val = values[0]
266
        else:
267
          # let's make sure there are no fields declared (cannot map >
268
          # 1 field without a function)
269
          assert not values, "LVM storage has multi-fields without a function"
270
          val = mapper
271

  
272
        row.append(val)
273

  
274
      data.append(row)
275

  
276
    return data
277

  
278
  @staticmethod
279
  def _BuildListCommand(cmd, sep, options, name):
280
    """Builds LVM command line.
281

  
282
    @type cmd: string
283
    @param cmd: Command name
284
    @type sep: string
285
    @param sep: Field separator character
286
    @type options: list of strings
287
    @param options: Wanted LVM fields
288
    @type name: name or None
289
    @param name: Name of requested entity
290

  
291
    """
292
    args = [cmd,
293
            "--noheadings", "--units=m", "--nosuffix",
294
            "--separator", sep,
295
            "--options", ",".join(options)]
296

  
297
    if name is not None:
298
      args.append(name)
299

  
300
    return args
301

  
302
  @staticmethod
303
  def _RunListCommand(args):
304
    """Run LVM command.
305

  
306
    """
307
    result = utils.RunCmd(args)
308

  
309
    if result.failed:
310
      raise errors.StorageError("Failed to run %r, command output: %s" %
311
                                (args[0], result.output))
312

  
313
    return result.stdout
314

  
315
  @staticmethod
316
  def _SplitList(data, sep, fieldcount):
317
    """Splits LVM command output into rows and fields.
318

  
319
    @type data: string
320
    @param data: LVM command output
321
    @type sep: string
322
    @param sep: Field separator character
323
    @type fieldcount: int
324
    @param fieldcount: Expected number of fields
325

  
326
    """
327
    for line in data.splitlines():
328
      fields = line.strip().split(sep)
329

  
330
      if len(fields) != fieldcount:
331
        logging.warning("Invalid line returned from lvm command: %s", line)
332
        continue
333

  
334
      yield fields
335

  
336

  
337
def _LvmPvGetAllocatable(attr):
338
  """Determines whether LVM PV is allocatable.
339

  
340
  @rtype: bool
341

  
342
  """
343
  if attr:
344
    return (attr[0] == "a")
345
  else:
346
    logging.warning("Invalid PV attribute: %r", attr)
347
    return False
348

  
349

  
350
class LvmPvStorage(_LvmBase): # pylint: disable=W0223
351
  """LVM Physical Volume storage unit.
352

  
353
  """
354
  LIST_COMMAND = "pvs"
355

  
356
  # Make sure to update constants.VALID_STORAGE_FIELDS when changing field
357
  # definitions.
358
  LIST_FIELDS = [
359
    (constants.SF_NAME, ["pv_name"], None),
360
    (constants.SF_SIZE, ["pv_size"], _ParseSize),
361
    (constants.SF_USED, ["pv_used"], _ParseSize),
362
    (constants.SF_FREE, ["pv_free"], _ParseSize),
363
    (constants.SF_ALLOCATABLE, ["pv_attr"], _LvmPvGetAllocatable),
364
    ]
365

  
366
  def _SetAllocatable(self, name, allocatable):
367
    """Sets the "allocatable" flag on a physical volume.
368

  
369
    @type name: string
370
    @param name: Physical volume name
371
    @type allocatable: bool
372
    @param allocatable: Whether to set the "allocatable" flag
373

  
374
    """
375
    args = ["pvchange", "--allocatable"]
376

  
377
    if allocatable:
378
      args.append("y")
379
    else:
380
      args.append("n")
381

  
382
    args.append(name)
383

  
384
    result = utils.RunCmd(args)
385
    if result.failed:
386
      raise errors.StorageError("Failed to modify physical volume,"
387
                                " pvchange output: %s" %
388
                                result.output)
389

  
390
  def Modify(self, name, changes):
391
    """Modifies flags on a physical volume.
392

  
393
    See L{_Base.Modify}.
394

  
395
    """
396
    if constants.SF_ALLOCATABLE in changes:
397
      self._SetAllocatable(name, changes[constants.SF_ALLOCATABLE])
398
      del changes[constants.SF_ALLOCATABLE]
399

  
400
    # Other changes will be handled (and maybe refused) by the base class.
401
    return _LvmBase.Modify(self, name, changes)
402

  
403

  
404
class LvmVgStorage(_LvmBase):
405
  """LVM Volume Group storage unit.
406

  
407
  """
408
  LIST_COMMAND = "vgs"
409
  VGREDUCE_COMMAND = "vgreduce"
410

  
411
  # Make sure to update constants.VALID_STORAGE_FIELDS when changing field
412
  # definitions.
413
  LIST_FIELDS = [
414
    (constants.SF_NAME, ["vg_name"], None),
415
    (constants.SF_SIZE, ["vg_size"], _ParseSize),
416
    (constants.SF_FREE, ["vg_free"], _ParseSize),
417
    (constants.SF_USED, ["vg_size", "vg_free"],
418
     lambda x, y: _ParseSize(x) - _ParseSize(y)),
419
    (constants.SF_ALLOCATABLE, [], True),
420
    ]
421

  
422
  def _RemoveMissing(self, name, _runcmd_fn=utils.RunCmd):
423
    """Runs "vgreduce --removemissing" on a volume group.
424

  
425
    @type name: string
426
    @param name: Volume group name
427

  
428
    """
429
    # Ignoring vgreduce exit code. Older versions exit with an error even tough
430
    # the VG is already consistent. This was fixed in later versions, but we
431
    # cannot depend on it.
432
    result = _runcmd_fn([self.VGREDUCE_COMMAND, "--removemissing", name])
433

  
434
    # Keep output in case something went wrong
435
    vgreduce_output = result.output
436

  
437
    # work around newer LVM version
438
    if ("Wrote out consistent volume group" not in vgreduce_output or
439
        "vgreduce --removemissing --force" in vgreduce_output):
440
      # we need to re-run with --force
441
      result = _runcmd_fn([self.VGREDUCE_COMMAND, "--removemissing",
442
                           "--force", name])
443
      vgreduce_output += "\n" + result.output
444

  
445
    result = _runcmd_fn([self.LIST_COMMAND, "--noheadings",
446
                         "--nosuffix", name])
447
    # we also need to check the output
448
    if result.failed or "Couldn't find device with uuid" in result.output:
449
      raise errors.StorageError(("Volume group '%s' still not consistent,"
450
                                 " 'vgreduce' output: %r,"
451
                                 " 'vgs' output: %r") %
452
                                (name, vgreduce_output, result.output))
453

  
454
  def Execute(self, name, op):
455
    """Executes an operation on a virtual volume.
456

  
457
    See L{_Base.Execute}.
458

  
459
    """
460
    if op == constants.SO_FIX_CONSISTENCY:
461
      return self._RemoveMissing(name)
462

  
463
    return _LvmBase.Execute(self, name, op)
464

  
465

  
466
# Lookup table for storage types
467
_STORAGE_TYPES = {
468
  constants.ST_FILE: FileStorage,
469
  constants.ST_LVM_PV: LvmPvStorage,
470
  constants.ST_LVM_VG: LvmVgStorage,
471
  }
472

  
473

  
474
def GetStorageClass(name):
475
  """Returns the class for a storage type.
476

  
477
  @type name: string
478
  @param name: Storage type
479

  
480
  """
481
  try:
482
    return _STORAGE_TYPES[name]
483
  except KeyError:
484
    raise errors.StorageError("Unknown storage type: %r" % name)
485

  
486

  
487
def GetStorage(name, *args):
488
  """Factory function for storage methods.
489

  
490
  @type name: string
491
  @param name: Storage type
492

  
493
  """
494
  return GetStorageClass(name)(*args)
b/lib/server/noded.py
45 45
from ganeti import daemon
46 46
from ganeti import http
47 47
from ganeti import utils
48
from ganeti import storage
48
from ganeti import container
49 49
from ganeti import serializer
50 50
from ganeti import netutils
51 51
from ganeti import pathutils
......
532 532

  
533 533
    """
534 534
    (su_name, su_args, name, fields) = params
535
    return storage.GetStorage(su_name, *su_args).List(name, fields)
535
    return container.GetStorage(su_name, *su_args).List(name, fields)
536 536

  
537 537
  @staticmethod
538 538
  def perspective_storage_modify(params):
......
540 540

  
541 541
    """
542 542
    (su_name, su_args, name, changes) = params
543
    return storage.GetStorage(su_name, *su_args).Modify(name, changes)
543
    return container.GetStorage(su_name, *su_args).Modify(name, changes)
544 544

  
545 545
  @staticmethod
546 546
  def perspective_storage_execute(params):
......
548 548

  
549 549
    """
550 550
    (su_name, su_args, name, op) = params
551
    return storage.GetStorage(su_name, *su_args).Execute(name, op)
551
    return container.GetStorage(su_name, *su_args).Execute(name, op)
552 552

  
553 553
  # bridge  --------------------------
554 554

  
/dev/null
1
#
2
#
3

  
4
# Copyright (C) 2009, 2011, 2012 Google Inc.
5
#
6
# This program is free software; you can redistribute it and/or modify
7
# it under the terms of the GNU General Public License as published by
8
# the Free Software Foundation; either version 2 of the License, or
9
# (at your option) any later version.
10
#
11
# This program is distributed in the hope that it will be useful, but
12
# WITHOUT ANY WARRANTY; without even the implied warranty of
13
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
14
# General Public License for more details.
15
#
16
# You should have received a copy of the GNU General Public License
17
# along with this program; if not, write to the Free Software
18
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
19
# 02110-1301, USA.
20

  
21

  
22
"""Storage container abstraction.
23

  
24
"""
25

  
26
# pylint: disable=W0232,R0201
27

  
28
# W0232, since we use these as singletons rather than object holding
29
# data
30

  
31
# R0201, for the same reason
32

  
33
# TODO: FileStorage initialised with paths whereas the others not
34

  
35
import logging
36

  
37
from ganeti import errors
38
from ganeti import constants
39
from ganeti import utils
40

  
41

  
42
def _ParseSize(value):
43
  return int(round(float(value), 0))
44

  
45

  
46
class _Base:
47
  """Base class for storage abstraction.
48

  
49
  """
50
  def List(self, name, fields):
51
    """Returns a list of all entities within the storage unit.
52

  
53
    @type name: string or None
54
    @param name: Entity name or None for all
55
    @type fields: list
56
    @param fields: List with all requested result fields (order is preserved)
57

  
58
    """
59
    raise NotImplementedError()
60

  
61
  def Modify(self, name, changes): # pylint: disable=W0613
62
    """Modifies an entity within the storage unit.
63

  
64
    @type name: string
65
    @param name: Entity name
66
    @type changes: dict
67
    @param changes: New field values
68

  
69
    """
70
    # Don't raise an error if no changes are requested
71
    if changes:
72
      raise errors.ProgrammerError("Unable to modify the following"
73
                                   "fields: %r" % (changes.keys(), ))
74

  
75
  def Execute(self, name, op):
76
    """Executes an operation on an entity within the storage unit.
77

  
78
    @type name: string
79
    @param name: Entity name
80
    @type op: string
81
    @param op: Operation name
82

  
83
    """
84
    raise NotImplementedError()
85

  
86

  
87
class FileStorage(_Base): # pylint: disable=W0223
88
  """File storage unit.
89

  
90
  """
91
  def __init__(self, paths):
92
    """Initializes this class.
93

  
94
    @type paths: list
95
    @param paths: List of file storage paths
96

  
97
    """
98
    self._paths = paths
99

  
100
  def List(self, name, fields):
101
    """Returns a list of all entities within the storage unit.
102

  
103
    See L{_Base.List}.
104

  
105
    """
106
    rows = []
107

  
108
    if name is None:
109
      paths = self._paths
110
    else:
111
      paths = [name]
112

  
113
    for path in paths:
114
      rows.append(self._ListInner(path, fields))
115

  
116
    return rows
117

  
118
  @staticmethod
119
  def _ListInner(path, fields):
120
    """Gathers requested information from directory.
121

  
122
    @type path: string
123
    @param path: Path to directory
124
    @type fields: list
125
    @param fields: Requested fields
126

  
127
    """
128
    values = []
129

  
130
    # Pre-calculate information in case it's requested more than once
131
    if constants.SF_USED in fields:
132
      dirsize = utils.CalculateDirectorySize(path)
133
    else:
134
      dirsize = None
135

  
136
    if constants.SF_FREE in fields or constants.SF_SIZE in fields:
137
      fsstats = utils.GetFilesystemStats(path)
138
    else:
139
      fsstats = None
140

  
141
    # Make sure to update constants.VALID_STORAGE_FIELDS when changing fields.
142
    for field_name in fields:
143
      if field_name == constants.SF_NAME:
144
        values.append(path)
145

  
146
      elif field_name == constants.SF_USED:
147
        values.append(dirsize)
148

  
149
      elif field_name == constants.SF_FREE:
150
        values.append(fsstats[1])
151

  
152
      elif field_name == constants.SF_SIZE:
153
        values.append(fsstats[0])
154

  
155
      elif field_name == constants.SF_ALLOCATABLE:
156
        values.append(True)
157

  
158
      else:
159
        raise errors.StorageError("Unknown field: %r" % field_name)
160

  
161
    return values
162

  
163

  
164
class _LvmBase(_Base): # pylint: disable=W0223
165
  """Base class for LVM storage containers.
166

  
167
  @cvar LIST_FIELDS: list of tuples consisting of three elements: SF_*
168
      constants, lvm command output fields (list), and conversion
169
      function or static value (for static value, the lvm output field
170
      can be an empty list)
171

  
172
  """
173
  LIST_SEP = "|"
174
  LIST_COMMAND = None
175
  LIST_FIELDS = None
176

  
177
  def List(self, name, wanted_field_names):
178
    """Returns a list of all entities within the storage unit.
179

  
180
    See L{_Base.List}.
181

  
182
    """
183
    # Get needed LVM fields
184
    lvm_fields = self._GetLvmFields(self.LIST_FIELDS, wanted_field_names)
185

  
186
    # Build LVM command
187
    cmd_args = self._BuildListCommand(self.LIST_COMMAND, self.LIST_SEP,
188
                                      lvm_fields, name)
189

  
190
    # Run LVM command
191
    cmd_result = self._RunListCommand(cmd_args)
192

  
193
    # Split and rearrange LVM command output
194
    return self._BuildList(self._SplitList(cmd_result, self.LIST_SEP,
195
                                           len(lvm_fields)),
196
                           self.LIST_FIELDS,
197
                           wanted_field_names,
198
                           lvm_fields)
199

  
200
  @staticmethod
201
  def _GetLvmFields(fields_def, wanted_field_names):
202
    """Returns unique list of fields wanted from LVM command.
203

  
204
    @type fields_def: list
205
    @param fields_def: Field definitions
206
    @type wanted_field_names: list
207
    @param wanted_field_names: List of requested fields
208

  
209
    """
210
    field_to_idx = dict([(field_name, idx)
211
                         for (idx, (field_name, _, _)) in
212
                         enumerate(fields_def)])
213

  
214
    lvm_fields = []
215

  
216
    for field_name in wanted_field_names:
217
      try:
218
        idx = field_to_idx[field_name]
219
      except IndexError:
220
        raise errors.StorageError("Unknown field: %r" % field_name)
221

  
222
      (_, lvm_names, _) = fields_def[idx]
223

  
224
      lvm_fields.extend(lvm_names)
225

  
226
    return utils.UniqueSequence(lvm_fields)
227

  
228
  @classmethod
229
  def _BuildList(cls, cmd_result, fields_def, wanted_field_names, lvm_fields):
230
    """Builds the final result list.
231

  
232
    @type cmd_result: iterable
233
    @param cmd_result: Iterable of LVM command output (iterable of lists)
234
    @type fields_def: list
235
    @param fields_def: Field definitions
236
    @type wanted_field_names: list
237
    @param wanted_field_names: List of requested fields
238
    @type lvm_fields: list
239
    @param lvm_fields: LVM fields
240

  
241
    """
242
    lvm_name_to_idx = dict([(lvm_name, idx)
243
                           for (idx, lvm_name) in enumerate(lvm_fields)])
244
    field_to_idx = dict([(field_name, idx)
245
                         for (idx, (field_name, _, _)) in
246
                         enumerate(fields_def)])
247

  
248
    data = []
249
    for raw_data in cmd_result:
250
      row = []
251

  
252
      for field_name in wanted_field_names:
253
        (_, lvm_names, mapper) = fields_def[field_to_idx[field_name]]
254

  
255
        values = [raw_data[lvm_name_to_idx[i]] for i in lvm_names]
256

  
257
        if callable(mapper):
258
          # we got a function, call it with all the declared fields
259
          val = mapper(*values) # pylint: disable=W0142
260
        elif len(values) == 1:
261
          assert mapper is None, ("Invalid mapper value (neither callable"
262
                                  " nor None) for one-element fields")
263
          # we don't have a function, but we had a single field
264
          # declared, pass it unchanged
265
          val = values[0]
266
        else:
267
          # let's make sure there are no fields declared (cannot map >
268
          # 1 field without a function)
269
          assert not values, "LVM storage has multi-fields without a function"
270
          val = mapper
271

  
272
        row.append(val)
273

  
274
      data.append(row)
275

  
276
    return data
277

  
278
  @staticmethod
279
  def _BuildListCommand(cmd, sep, options, name):
280
    """Builds LVM command line.
281

  
282
    @type cmd: string
283
    @param cmd: Command name
284
    @type sep: string
285
    @param sep: Field separator character
286
    @type options: list of strings
287
    @param options: Wanted LVM fields
288
    @type name: name or None
289
    @param name: Name of requested entity
290

  
291
    """
292
    args = [cmd,
293
            "--noheadings", "--units=m", "--nosuffix",
294
            "--separator", sep,
295
            "--options", ",".join(options)]
296

  
297
    if name is not None:
298
      args.append(name)
299

  
300
    return args
301

  
302
  @staticmethod
303
  def _RunListCommand(args):
304
    """Run LVM command.
305

  
306
    """
307
    result = utils.RunCmd(args)
308

  
309
    if result.failed:
310
      raise errors.StorageError("Failed to run %r, command output: %s" %
311
                                (args[0], result.output))
312

  
313
    return result.stdout
314

  
315
  @staticmethod
316
  def _SplitList(data, sep, fieldcount):
317
    """Splits LVM command output into rows and fields.
318

  
319
    @type data: string
320
    @param data: LVM command output
321
    @type sep: string
322
    @param sep: Field separator character
323
    @type fieldcount: int
324
    @param fieldcount: Expected number of fields
325

  
326
    """
327
    for line in data.splitlines():
328
      fields = line.strip().split(sep)
329

  
330
      if len(fields) != fieldcount:
331
        logging.warning("Invalid line returned from lvm command: %s", line)
332
        continue
333

  
334
      yield fields
335

  
336

  
337
def _LvmPvGetAllocatable(attr):
338
  """Determines whether LVM PV is allocatable.
339

  
340
  @rtype: bool
341

  
342
  """
343
  if attr:
344
    return (attr[0] == "a")
345
  else:
346
    logging.warning("Invalid PV attribute: %r", attr)
347
    return False
348

  
349

  
350
class LvmPvStorage(_LvmBase): # pylint: disable=W0223
351
  """LVM Physical Volume storage unit.
352

  
353
  """
354
  LIST_COMMAND = "pvs"
355

  
356
  # Make sure to update constants.VALID_STORAGE_FIELDS when changing field
357
  # definitions.
358
  LIST_FIELDS = [
359
    (constants.SF_NAME, ["pv_name"], None),
360
    (constants.SF_SIZE, ["pv_size"], _ParseSize),
361
    (constants.SF_USED, ["pv_used"], _ParseSize),
362
    (constants.SF_FREE, ["pv_free"], _ParseSize),
363
    (constants.SF_ALLOCATABLE, ["pv_attr"], _LvmPvGetAllocatable),
364
    ]
365

  
366
  def _SetAllocatable(self, name, allocatable):
367
    """Sets the "allocatable" flag on a physical volume.
368

  
369
    @type name: string
370
    @param name: Physical volume name
371
    @type allocatable: bool
372
    @param allocatable: Whether to set the "allocatable" flag
373

  
374
    """
375
    args = ["pvchange", "--allocatable"]
376

  
377
    if allocatable:
378
      args.append("y")
379
    else:
380
      args.append("n")
381

  
382
    args.append(name)
383

  
384
    result = utils.RunCmd(args)
385
    if result.failed:
386
      raise errors.StorageError("Failed to modify physical volume,"
387
                                " pvchange output: %s" %
388
                                result.output)
389

  
390
  def Modify(self, name, changes):
391
    """Modifies flags on a physical volume.
392

  
393
    See L{_Base.Modify}.
394

  
395
    """
396
    if constants.SF_ALLOCATABLE in changes:
397
      self._SetAllocatable(name, changes[constants.SF_ALLOCATABLE])
398
      del changes[constants.SF_ALLOCATABLE]
399

  
400
    # Other changes will be handled (and maybe refused) by the base class.
401
    return _LvmBase.Modify(self, name, changes)
402

  
403

  
404
class LvmVgStorage(_LvmBase):
405
  """LVM Volume Group storage unit.
406

  
407
  """
408
  LIST_COMMAND = "vgs"
409
  VGREDUCE_COMMAND = "vgreduce"
410

  
411
  # Make sure to update constants.VALID_STORAGE_FIELDS when changing field
412
  # definitions.
413
  LIST_FIELDS = [
414
    (constants.SF_NAME, ["vg_name"], None),
415
    (constants.SF_SIZE, ["vg_size"], _ParseSize),
416
    (constants.SF_FREE, ["vg_free"], _ParseSize),
417
    (constants.SF_USED, ["vg_size", "vg_free"],
418
     lambda x, y: _ParseSize(x) - _ParseSize(y)),
419
    (constants.SF_ALLOCATABLE, [], True),
420
    ]
421

  
422
  def _RemoveMissing(self, name, _runcmd_fn=utils.RunCmd):
423
    """Runs "vgreduce --removemissing" on a volume group.
424

  
425
    @type name: string
426
    @param name: Volume group name
427

  
428
    """
429
    # Ignoring vgreduce exit code. Older versions exit with an error even tough
430
    # the VG is already consistent. This was fixed in later versions, but we
431
    # cannot depend on it.
432
    result = _runcmd_fn([self.VGREDUCE_COMMAND, "--removemissing", name])
433

  
434
    # Keep output in case something went wrong
435
    vgreduce_output = result.output
436

  
437
    # work around newer LVM version
438
    if ("Wrote out consistent volume group" not in vgreduce_output or
439
        "vgreduce --removemissing --force" in vgreduce_output):
440
      # we need to re-run with --force
441
      result = _runcmd_fn([self.VGREDUCE_COMMAND, "--removemissing",
442
                           "--force", name])
443
      vgreduce_output += "\n" + result.output
444

  
445
    result = _runcmd_fn([self.LIST_COMMAND, "--noheadings",
446
                         "--nosuffix", name])
447
    # we also need to check the output
448
    if result.failed or "Couldn't find device with uuid" in result.output:
449
      raise errors.StorageError(("Volume group '%s' still not consistent,"
450
                                 " 'vgreduce' output: %r,"
451
                                 " 'vgs' output: %r") %
452
                                (name, vgreduce_output, result.output))
453

  
454
  def Execute(self, name, op):
455
    """Executes an operation on a virtual volume.
456

  
457
    See L{_Base.Execute}.
458

  
459
    """
460
    if op == constants.SO_FIX_CONSISTENCY:
461
      return self._RemoveMissing(name)
462

  
463
    return _LvmBase.Execute(self, name, op)
464

  
465

  
466
# Lookup table for storage types
467
_STORAGE_TYPES = {
468
  constants.ST_FILE: FileStorage,
469
  constants.ST_LVM_PV: LvmPvStorage,
470
  constants.ST_LVM_VG: LvmVgStorage,
471
  }
472

  
473

  
474
def GetStorageClass(name):
475
  """Returns the class for a storage type.
476

  
477
  @type name: string
478
  @param name: Storage type
479

  
480
  """
481
  try:
482
    return _STORAGE_TYPES[name]
483
  except KeyError:
484
    raise errors.StorageError("Unknown storage type: %r" % name)
485

  
486

  
487
def GetStorage(name, *args):
488
  """Factory function for storage methods.
489

  
490
  @type name: string
491
  @param name: Storage type
492

  
493
  """
494
  return GetStorageClass(name)(*args)
b/test/py/ganeti.container_unittest.py
1
#!/usr/bin/python
2
#
3

  
4
# Copyright (C) 2012 Google Inc.
5
#
6
# This program is free software; you can redistribute it and/or modify
7
# it under the terms of the GNU General Public License as published by
8
# the Free Software Foundation; either version 2 of the License, or
9
# (at your option) any later version.
10
#
11
# This program is distributed in the hope that it will be useful, but
12
# WITHOUT ANY WARRANTY; without even the implied warranty of
13
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
14
# General Public License for more details.
15
#
16
# You should have received a copy of the GNU General Public License
17
# along with this program; if not, write to the Free Software
18
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
19
# 02110-1301, USA.
20

  
21

  
22
"""Script for testing ganeti.storage.container"""
23

  
24
import re
25
import unittest
26
import random
27

  
28
from ganeti import constants
29
from ganeti import utils
30
from ganeti import compat
31
from ganeti import errors
32
from ganeti import container
33

  
34
import testutils
35

  
36

  
37
class TestVGReduce(testutils.GanetiTestCase):
38
  VGNAME = "xenvg"
39
  LIST_CMD = container.LvmVgStorage.LIST_COMMAND
40
  VGREDUCE_CMD = container.LvmVgStorage.VGREDUCE_COMMAND
41

  
42
  def _runCmd(self, cmd, **kwargs):
43
    if not self.run_history:
44
      self.fail("Empty run results")
45
    exp_cmd, result = self.run_history.pop(0)
46
    self.assertEqual(cmd, exp_cmd)
47
    return result
48

  
49
  def testOldVersion(self):
50
    lvmvg = container.LvmVgStorage()
51
    stdout = testutils.ReadTestData("vgreduce-removemissing-2.02.02.txt")
52
    vgs_fail = testutils.ReadTestData("vgs-missing-pvs-2.02.02.txt")
53
    self.run_history = [
54
      ([self.VGREDUCE_CMD, "--removemissing", self.VGNAME],
55
       utils.RunResult(0, None, stdout, "", "", None, None)),
56
      ([self.LIST_CMD, "--noheadings", "--nosuffix", self.VGNAME],
57
       utils.RunResult(0, None, "", "", "", None, None)),
58
      ]
59
    lvmvg._RemoveMissing(self.VGNAME, _runcmd_fn=self._runCmd)
60
    self.assertEqual(self.run_history, [])
61
    for ecode, out in [(1, ""), (0, vgs_fail)]:
62
      self.run_history = [
63
        ([self.VGREDUCE_CMD, "--removemissing", self.VGNAME],
64
         utils.RunResult(0, None, stdout, "", "", None, None)),
65
        ([self.LIST_CMD, "--noheadings", "--nosuffix", self.VGNAME],
66
         utils.RunResult(ecode, None, out, "", "", None, None)),
67
        ]
68
      self.assertRaises(errors.StorageError, lvmvg._RemoveMissing, self.VGNAME,
69
                        _runcmd_fn=self._runCmd)
70
      self.assertEqual(self.run_history, [])
71

  
72
  def testNewVersion(self):
73
    lvmvg = container.LvmVgStorage()
74
    stdout1 = testutils.ReadTestData("vgreduce-removemissing-2.02.66-fail.txt")
75
    stdout2 = testutils.ReadTestData("vgreduce-removemissing-2.02.66-ok.txt")
76
    vgs_fail = testutils.ReadTestData("vgs-missing-pvs-2.02.66.txt")
77
    # first: require --fail, check that it's used
78
    self.run_history = [
79
      ([self.VGREDUCE_CMD, "--removemissing", self.VGNAME],
80
       utils.RunResult(0, None, stdout1, "", "", None, None)),
81
      ([self.VGREDUCE_CMD, "--removemissing", "--force", self.VGNAME],
82
       utils.RunResult(0, None, stdout2, "", "", None, None)),
83
      ([self.LIST_CMD, "--noheadings", "--nosuffix", self.VGNAME],
84
       utils.RunResult(0, None, "", "", "", None, None)),
85
      ]
86
    lvmvg._RemoveMissing(self.VGNAME, _runcmd_fn=self._runCmd)
87
    self.assertEqual(self.run_history, [])
88
    # second: make sure --fail is not used if not needed
89
    self.run_history = [
90
      ([self.VGREDUCE_CMD, "--removemissing", self.VGNAME],
91
       utils.RunResult(0, None, stdout2, "", "", None, None)),
92
      ([self.LIST_CMD, "--noheadings", "--nosuffix", self.VGNAME],
93
       utils.RunResult(0, None, "", "", "", None, None)),
94
      ]
95
    lvmvg._RemoveMissing(self.VGNAME, _runcmd_fn=self._runCmd)
96
    self.assertEqual(self.run_history, [])
97
    # third: make sure we error out if vgs doesn't find the volume
98
    for ecode, out in [(1, ""), (0, vgs_fail)]:
99
      self.run_history = [
100
        ([self.VGREDUCE_CMD, "--removemissing", self.VGNAME],
101
         utils.RunResult(0, None, stdout1, "", "", None, None)),
102
        ([self.VGREDUCE_CMD, "--removemissing", "--force", self.VGNAME],
103
         utils.RunResult(0, None, stdout2, "", "", None, None)),
104
        ([self.LIST_CMD, "--noheadings", "--nosuffix", self.VGNAME],
105
         utils.RunResult(ecode, None, out, "", "", None, None)),
106
        ]
107
      self.assertRaises(errors.StorageError, lvmvg._RemoveMissing, self.VGNAME,
108
                        _runcmd_fn=self._runCmd)
109
      self.assertEqual(self.run_history, [])
110

  
111

  
112
if __name__ == "__main__":
113
  testutils.GanetiTestProgram()
/dev/null
1
#!/usr/bin/python
2
#
3

  
4
# Copyright (C) 2012 Google Inc.
5
#
6
# This program is free software; you can redistribute it and/or modify
7
# it under the terms of the GNU General Public License as published by
8
# the Free Software Foundation; either version 2 of the License, or
9
# (at your option) any later version.
10
#
11
# This program is distributed in the hope that it will be useful, but
12
# WITHOUT ANY WARRANTY; without even the implied warranty of
13
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
14
# General Public License for more details.
15
#
16
# You should have received a copy of the GNU General Public License
17
# along with this program; if not, write to the Free Software
18
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
19
# 02110-1301, USA.
20

  
21

  
22
"""Script for testing ganeti.storage"""
23

  
24
import re
25
import unittest
26
import random
27

  
28
from ganeti import constants
29
from ganeti import utils
30
from ganeti import compat
31
from ganeti import errors
32
from ganeti import storage
33

  
34
import testutils
35

  
36

  
37
class TestVGReduce(testutils.GanetiTestCase):
38
  VGNAME = "xenvg"
39
  LIST_CMD = storage.LvmVgStorage.LIST_COMMAND
40
  VGREDUCE_CMD = storage.LvmVgStorage.VGREDUCE_COMMAND
41

  
42
  def _runCmd(self, cmd, **kwargs):
43
    if not self.run_history:
44
      self.fail("Empty run results")
45
    exp_cmd, result = self.run_history.pop(0)
46
    self.assertEqual(cmd, exp_cmd)
47
    return result
48

  
49
  def testOldVersion(self):
50
    lvmvg = storage.LvmVgStorage()
51
    stdout = testutils.ReadTestData("vgreduce-removemissing-2.02.02.txt")
52
    vgs_fail = testutils.ReadTestData("vgs-missing-pvs-2.02.02.txt")
53
    self.run_history = [
54
      ([self.VGREDUCE_CMD, "--removemissing", self.VGNAME],
55
       utils.RunResult(0, None, stdout, "", "", None, None)),
56
      ([self.LIST_CMD, "--noheadings", "--nosuffix", self.VGNAME],
57
       utils.RunResult(0, None, "", "", "", None, None)),
58
      ]
59
    lvmvg._RemoveMissing(self.VGNAME, _runcmd_fn=self._runCmd)
60
    self.assertEqual(self.run_history, [])
61
    for ecode, out in [(1, ""), (0, vgs_fail)]:
62
      self.run_history = [
63
        ([self.VGREDUCE_CMD, "--removemissing", self.VGNAME],
64
         utils.RunResult(0, None, stdout, "", "", None, None)),
65
        ([self.LIST_CMD, "--noheadings", "--nosuffix", self.VGNAME],
66
         utils.RunResult(ecode, None, out, "", "", None, None)),
67
        ]
68
      self.assertRaises(errors.StorageError, lvmvg._RemoveMissing, self.VGNAME,
69
                        _runcmd_fn=self._runCmd)
70
      self.assertEqual(self.run_history, [])
71

  
72
  def testNewVersion(self):
73
    lvmvg = storage.LvmVgStorage()
74
    stdout1 = testutils.ReadTestData("vgreduce-removemissing-2.02.66-fail.txt")
75
    stdout2 = testutils.ReadTestData("vgreduce-removemissing-2.02.66-ok.txt")
76
    vgs_fail = testutils.ReadTestData("vgs-missing-pvs-2.02.66.txt")
77
    # first: require --fail, check that it's used
78
    self.run_history = [
79
      ([self.VGREDUCE_CMD, "--removemissing", self.VGNAME],
80
       utils.RunResult(0, None, stdout1, "", "", None, None)),
81
      ([self.VGREDUCE_CMD, "--removemissing", "--force", self.VGNAME],
82
       utils.RunResult(0, None, stdout2, "", "", None, None)),
83
      ([self.LIST_CMD, "--noheadings", "--nosuffix", self.VGNAME],
84
       utils.RunResult(0, None, "", "", "", None, None)),
85
      ]
86
    lvmvg._RemoveMissing(self.VGNAME, _runcmd_fn=self._runCmd)
87
    self.assertEqual(self.run_history, [])
88
    # second: make sure --fail is not used if not needed
89
    self.run_history = [
90
      ([self.VGREDUCE_CMD, "--removemissing", self.VGNAME],
91
       utils.RunResult(0, None, stdout2, "", "", None, None)),
92
      ([self.LIST_CMD, "--noheadings", "--nosuffix", self.VGNAME],
93
       utils.RunResult(0, None, "", "", "", None, None)),
94
      ]
95
    lvmvg._RemoveMissing(self.VGNAME, _runcmd_fn=self._runCmd)
96
    self.assertEqual(self.run_history, [])
97
    # third: make sure we error out if vgs doesn't find the volume
98
    for ecode, out in [(1, ""), (0, vgs_fail)]:
99
      self.run_history = [
100
        ([self.VGREDUCE_CMD, "--removemissing", self.VGNAME],
101
         utils.RunResult(0, None, stdout1, "", "", None, None)),
102
        ([self.VGREDUCE_CMD, "--removemissing", "--force", self.VGNAME],
103
         utils.RunResult(0, None, stdout2, "", "", None, None)),
104
        ([self.LIST_CMD, "--noheadings", "--nosuffix", self.VGNAME],
105
         utils.RunResult(ecode, None, out, "", "", None, None)),
106
        ]
107
      self.assertRaises(errors.StorageError, lvmvg._RemoveMissing, self.VGNAME,
108
                        _runcmd_fn=self._runCmd)
109
      self.assertEqual(self.run_history, [])
110

  
111

  
112
if __name__ == "__main__":
113
  testutils.GanetiTestProgram()

Also available in: Unified diff