Statistics
| Branch: | Tag: | Revision:

root / lib / storage.py @ 91c17910

History | View | Annotate | Download (13.2 kB)

1
#
2
#
3

    
4
# Copyright (C) 2009, 2011, 2012 Google Inc.
5
#
6
# This program is free software; you can redistribute it and/or modify
7
# it under the terms of the GNU General Public License as published by
8
# the Free Software Foundation; either version 2 of the License, or
9
# (at your option) any later version.
10
#
11
# This program is distributed in the hope that it will be useful, but
12
# WITHOUT ANY WARRANTY; without even the implied warranty of
13
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
14
# General Public License for more details.
15
#
16
# You should have received a copy of the GNU General Public License
17
# along with this program; if not, write to the Free Software
18
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
19
# 02110-1301, USA.
20

    
21

    
22
"""Storage container abstraction.
23

24
"""
25

    
26
# pylint: disable=W0232,R0201
27

    
28
# W0232, since we use these as singletons rather than object holding
29
# data
30

    
31
# R0201, for the same reason
32

    
33
# TODO: FileStorage initialised with paths whereas the others not
34

    
35
import logging
36

    
37
from ganeti import errors
38
from ganeti import constants
39
from ganeti import utils
40

    
41

    
42
def _ParseSize(value):
43
  return int(round(float(value), 0))
44

    
45

    
46
class _Base:
47
  """Base class for storage abstraction.
48

49
  """
50
  def List(self, name, fields):
51
    """Returns a list of all entities within the storage unit.
52

53
    @type name: string or None
54
    @param name: Entity name or None for all
55
    @type fields: list
56
    @param fields: List with all requested result fields (order is preserved)
57

58
    """
59
    raise NotImplementedError()
60

    
61
  def Modify(self, name, changes): # pylint: disable=W0613
62
    """Modifies an entity within the storage unit.
63

64
    @type name: string
65
    @param name: Entity name
66
    @type changes: dict
67
    @param changes: New field values
68

69
    """
70
    # Don't raise an error if no changes are requested
71
    if changes:
72
      raise errors.ProgrammerError("Unable to modify the following"
73
                                   "fields: %r" % (changes.keys(), ))
74

    
75
  def Execute(self, name, op):
76
    """Executes an operation on an entity within the storage unit.
77

78
    @type name: string
79
    @param name: Entity name
80
    @type op: string
81
    @param op: Operation name
82

83
    """
84
    raise NotImplementedError()
85

    
86

    
87
class FileStorage(_Base): # pylint: disable=W0223
88
  """File storage unit.
89

90
  """
91
  def __init__(self, paths):
92
    """Initializes this class.
93

94
    @type paths: list
95
    @param paths: List of file storage paths
96

97
    """
98
    self._paths = paths
99

    
100
  def List(self, name, fields):
101
    """Returns a list of all entities within the storage unit.
102

103
    See L{_Base.List}.
104

105
    """
106
    rows = []
107

    
108
    if name is None:
109
      paths = self._paths
110
    else:
111
      paths = [name]
112

    
113
    for path in paths:
114
      rows.append(self._ListInner(path, fields))
115

    
116
    return rows
117

    
118
  @staticmethod
119
  def _ListInner(path, fields):
120
    """Gathers requested information from directory.
121

122
    @type path: string
123
    @param path: Path to directory
124
    @type fields: list
125
    @param fields: Requested fields
126

127
    """
128
    values = []
129

    
130
    # Pre-calculate information in case it's requested more than once
131
    if constants.SF_USED in fields:
132
      dirsize = utils.CalculateDirectorySize(path)
133
    else:
134
      dirsize = None
135

    
136
    if constants.SF_FREE in fields or constants.SF_SIZE in fields:
137
      fsstats = utils.GetFilesystemStats(path)
138
    else:
139
      fsstats = None
140

    
141
    # Make sure to update constants.VALID_STORAGE_FIELDS when changing fields.
142
    for field_name in fields:
143
      if field_name == constants.SF_NAME:
144
        values.append(path)
145

    
146
      elif field_name == constants.SF_USED:
147
        values.append(dirsize)
148

    
149
      elif field_name == constants.SF_FREE:
150
        values.append(fsstats[1])
151

    
152
      elif field_name == constants.SF_SIZE:
153
        values.append(fsstats[0])
154

    
155
      elif field_name == constants.SF_ALLOCATABLE:
156
        values.append(True)
157

    
158
      else:
159
        raise errors.StorageError("Unknown field: %r" % field_name)
160

    
161
    return values
162

    
163

    
164
class _LvmBase(_Base): # pylint: disable=W0223
165
  """Base class for LVM storage containers.
166

167
  @cvar LIST_FIELDS: list of tuples consisting of three elements: SF_*
168
      constants, lvm command output fields (list), and conversion
169
      function or static value (for static value, the lvm output field
170
      can be an empty list)
171

172
  """
173
  LIST_SEP = "|"
174
  LIST_COMMAND = None
175
  LIST_FIELDS = None
176

    
177
  def List(self, name, wanted_field_names):
178
    """Returns a list of all entities within the storage unit.
179

180
    See L{_Base.List}.
181

182
    """
183
    # Get needed LVM fields
184
    lvm_fields = self._GetLvmFields(self.LIST_FIELDS, wanted_field_names)
185

    
186
    # Build LVM command
187
    cmd_args = self._BuildListCommand(self.LIST_COMMAND, self.LIST_SEP,
188
                                      lvm_fields, name)
189

    
190
    # Run LVM command
191
    cmd_result = self._RunListCommand(cmd_args)
192

    
193
    # Split and rearrange LVM command output
194
    return self._BuildList(self._SplitList(cmd_result, self.LIST_SEP,
195
                                           len(lvm_fields)),
196
                           self.LIST_FIELDS,
197
                           wanted_field_names,
198
                           lvm_fields)
199

    
200
  @staticmethod
201
  def _GetLvmFields(fields_def, wanted_field_names):
202
    """Returns unique list of fields wanted from LVM command.
203

204
    @type fields_def: list
205
    @param fields_def: Field definitions
206
    @type wanted_field_names: list
207
    @param wanted_field_names: List of requested fields
208

209
    """
210
    field_to_idx = dict([(field_name, idx)
211
                         for (idx, (field_name, _, _)) in
212
                         enumerate(fields_def)])
213

    
214
    lvm_fields = []
215

    
216
    for field_name in wanted_field_names:
217
      try:
218
        idx = field_to_idx[field_name]
219
      except IndexError:
220
        raise errors.StorageError("Unknown field: %r" % field_name)
221

    
222
      (_, lvm_names, _) = fields_def[idx]
223

    
224
      lvm_fields.extend(lvm_names)
225

    
226
    return utils.UniqueSequence(lvm_fields)
227

    
228
  @classmethod
229
  def _BuildList(cls, cmd_result, fields_def, wanted_field_names, lvm_fields):
230
    """Builds the final result list.
231

232
    @type cmd_result: iterable
233
    @param cmd_result: Iterable of LVM command output (iterable of lists)
234
    @type fields_def: list
235
    @param fields_def: Field definitions
236
    @type wanted_field_names: list
237
    @param wanted_field_names: List of requested fields
238
    @type lvm_fields: list
239
    @param lvm_fields: LVM fields
240

241
    """
242
    lvm_name_to_idx = dict([(lvm_name, idx)
243
                           for (idx, lvm_name) in enumerate(lvm_fields)])
244
    field_to_idx = dict([(field_name, idx)
245
                         for (idx, (field_name, _, _)) in
246
                         enumerate(fields_def)])
247

    
248
    data = []
249
    for raw_data in cmd_result:
250
      row = []
251

    
252
      for field_name in wanted_field_names:
253
        (_, lvm_names, mapper) = fields_def[field_to_idx[field_name]]
254

    
255
        values = [raw_data[lvm_name_to_idx[i]] for i in lvm_names]
256

    
257
        if callable(mapper):
258
          # we got a function, call it with all the declared fields
259
          val = mapper(*values) # pylint: disable=W0142
260
        elif len(values) == 1:
261
          assert mapper is None, ("Invalid mapper value (neither callable"
262
                                  " nor None) for one-element fields")
263
          # we don't have a function, but we had a single field
264
          # declared, pass it unchanged
265
          val = values[0]
266
        else:
267
          # let's make sure there are no fields declared (cannot map >
268
          # 1 field without a function)
269
          assert not values, "LVM storage has multi-fields without a function"
270
          val = mapper
271

    
272
        row.append(val)
273

    
274
      data.append(row)
275

    
276
    return data
277

    
278
  @staticmethod
279
  def _BuildListCommand(cmd, sep, options, name):
280
    """Builds LVM command line.
281

282
    @type cmd: string
283
    @param cmd: Command name
284
    @type sep: string
285
    @param sep: Field separator character
286
    @type options: list of strings
287
    @param options: Wanted LVM fields
288
    @type name: name or None
289
    @param name: Name of requested entity
290

291
    """
292
    args = [cmd,
293
            "--noheadings", "--units=m", "--nosuffix",
294
            "--separator", sep,
295
            "--options", ",".join(options)]
296

    
297
    if name is not None:
298
      args.append(name)
299

    
300
    return args
301

    
302
  @staticmethod
303
  def _RunListCommand(args):
304
    """Run LVM command.
305

306
    """
307
    result = utils.RunCmd(args)
308

    
309
    if result.failed:
310
      raise errors.StorageError("Failed to run %r, command output: %s" %
311
                                (args[0], result.output))
312

    
313
    return result.stdout
314

    
315
  @staticmethod
316
  def _SplitList(data, sep, fieldcount):
317
    """Splits LVM command output into rows and fields.
318

319
    @type data: string
320
    @param data: LVM command output
321
    @type sep: string
322
    @param sep: Field separator character
323
    @type fieldcount: int
324
    @param fieldcount: Expected number of fields
325

326
    """
327
    for line in data.splitlines():
328
      fields = line.strip().split(sep)
329

    
330
      if len(fields) != fieldcount:
331
        logging.warning("Invalid line returned from lvm command: %s", line)
332
        continue
333

    
334
      yield fields
335

    
336

    
337
def _LvmPvGetAllocatable(attr):
338
  """Determines whether LVM PV is allocatable.
339

340
  @rtype: bool
341

342
  """
343
  if attr:
344
    return (attr[0] == "a")
345
  else:
346
    logging.warning("Invalid PV attribute: %r", attr)
347
    return False
348

    
349

    
350
class LvmPvStorage(_LvmBase): # pylint: disable=W0223
351
  """LVM Physical Volume storage unit.
352

353
  """
354
  LIST_COMMAND = "pvs"
355

    
356
  # Make sure to update constants.VALID_STORAGE_FIELDS when changing field
357
  # definitions.
358
  LIST_FIELDS = [
359
    (constants.SF_NAME, ["pv_name"], None),
360
    (constants.SF_SIZE, ["pv_size"], _ParseSize),
361
    (constants.SF_USED, ["pv_used"], _ParseSize),
362
    (constants.SF_FREE, ["pv_free"], _ParseSize),
363
    (constants.SF_ALLOCATABLE, ["pv_attr"], _LvmPvGetAllocatable),
364
    ]
365

    
366
  def _SetAllocatable(self, name, allocatable):
367
    """Sets the "allocatable" flag on a physical volume.
368

369
    @type name: string
370
    @param name: Physical volume name
371
    @type allocatable: bool
372
    @param allocatable: Whether to set the "allocatable" flag
373

374
    """
375
    args = ["pvchange", "--allocatable"]
376

    
377
    if allocatable:
378
      args.append("y")
379
    else:
380
      args.append("n")
381

    
382
    args.append(name)
383

    
384
    result = utils.RunCmd(args)
385
    if result.failed:
386
      raise errors.StorageError("Failed to modify physical volume,"
387
                                " pvchange output: %s" %
388
                                result.output)
389

    
390
  def Modify(self, name, changes):
391
    """Modifies flags on a physical volume.
392

393
    See L{_Base.Modify}.
394

395
    """
396
    if constants.SF_ALLOCATABLE in changes:
397
      self._SetAllocatable(name, changes[constants.SF_ALLOCATABLE])
398
      del changes[constants.SF_ALLOCATABLE]
399

    
400
    # Other changes will be handled (and maybe refused) by the base class.
401
    return _LvmBase.Modify(self, name, changes)
402

    
403

    
404
class LvmVgStorage(_LvmBase):
405
  """LVM Volume Group storage unit.
406

407
  """
408
  LIST_COMMAND = "vgs"
409
  VGREDUCE_COMMAND = "vgreduce"
410

    
411
  # Make sure to update constants.VALID_STORAGE_FIELDS when changing field
412
  # definitions.
413
  LIST_FIELDS = [
414
    (constants.SF_NAME, ["vg_name"], None),
415
    (constants.SF_SIZE, ["vg_size"], _ParseSize),
416
    (constants.SF_FREE, ["vg_free"], _ParseSize),
417
    (constants.SF_USED, ["vg_size", "vg_free"],
418
     lambda x, y: _ParseSize(x) - _ParseSize(y)),
419
    (constants.SF_ALLOCATABLE, [], True),
420
    ]
421

    
422
  def _RemoveMissing(self, name, _runcmd_fn=utils.RunCmd):
423
    """Runs "vgreduce --removemissing" on a volume group.
424

425
    @type name: string
426
    @param name: Volume group name
427

428
    """
429
    # Ignoring vgreduce exit code. Older versions exit with an error even tough
430
    # the VG is already consistent. This was fixed in later versions, but we
431
    # cannot depend on it.
432
    result = _runcmd_fn([self.VGREDUCE_COMMAND, "--removemissing", name])
433

    
434
    # Keep output in case something went wrong
435
    vgreduce_output = result.output
436

    
437
    # work around newer LVM version
438
    if ("Wrote out consistent volume group" not in vgreduce_output or
439
        "vgreduce --removemissing --force" in vgreduce_output):
440
      # we need to re-run with --force
441
      result = _runcmd_fn([self.VGREDUCE_COMMAND, "--removemissing",
442
                           "--force", name])
443
      vgreduce_output += "\n" + result.output
444

    
445
    result = _runcmd_fn([self.LIST_COMMAND, "--noheadings",
446
                         "--nosuffix", name])
447
    # we also need to check the output
448
    if result.failed or "Couldn't find device with uuid" in result.output:
449
      raise errors.StorageError(("Volume group '%s' still not consistent,"
450
                                 " 'vgreduce' output: %r,"
451
                                 " 'vgs' output: %r") %
452
                                (name, vgreduce_output, result.output))
453

    
454
  def Execute(self, name, op):
455
    """Executes an operation on a virtual volume.
456

457
    See L{_Base.Execute}.
458

459
    """
460
    if op == constants.SO_FIX_CONSISTENCY:
461
      return self._RemoveMissing(name)
462

    
463
    return _LvmBase.Execute(self, name, op)
464

    
465

    
466
# Lookup table for storage types
467
_STORAGE_TYPES = {
468
  constants.ST_FILE: FileStorage,
469
  constants.ST_LVM_PV: LvmPvStorage,
470
  constants.ST_LVM_VG: LvmVgStorage,
471
  }
472

    
473

    
474
def GetStorageClass(name):
475
  """Returns the class for a storage type.
476

477
  @type name: string
478
  @param name: Storage type
479

480
  """
481
  try:
482
    return _STORAGE_TYPES[name]
483
  except KeyError:
484
    raise errors.StorageError("Unknown storage type: %r" % name)
485

    
486

    
487
def GetStorage(name, *args):
488
  """Factory function for storage methods.
489

490
  @type name: string
491
  @param name: Storage type
492

493
  """
494
  return GetStorageClass(name)(*args)