code.grnet.gr Git - archipelago/blob - xseg/archipelago

   1 #!/usr/bin/env python
   2 #
   3 # archipelagos tool
   4
   5 import os, sys, subprocess, argparse, time, psutil, signal, errno
   6 from subprocess import call, check_call
   7
   8 DEFAULTS='/etc/default/archipelago'
   9
  10 #system defaults
  11 PIDFILE_PATH="/var/run/archipelago"
  12 LOGS_PATH="/var/log/archipelago"
  13 DEVICE_PREFIX="/dev/xsegbd"
  14 XSEGBD_SYSFS="/sys/bus/xsegbd/"
  15
  16 CHARDEV_NAME="/dev/segdev"
  17 CHARDEV_MAJOR=60
  18 CHARDEV_MINOR=0
  19
  20 FILE_BLOCKER='mt-pfiled'
  21 RADOS_BLOCKER='mt-sosd'
  22 MAPPER='mt-mapperd'
  23 VLMC='st-vlmcd'
  24 BLOCKER=''
  25
  26 available_storage = {'files': FILE_BLOCKER, 'rados': RADOS_BLOCKER}
  27
  28 peers = []
  29 modules = ["xseg", "segdev", "xseg_posix", "xseg_pthread", "xseg_segdev"]
  30 xsegbd = "xsegbd"
  31
  32 BPORT=0
  33 MPORT=1
  34 MBPORT=2
  35 VTOOL=3
  36 VPORT_START=204
  37 VPORT_END=403
  38
  39 #default config
  40 SPEC="segdev:xsegbd:512:1024:12"
  41
  42 NR_OPS_BLOCKERB=""
  43 NR_OPS_BLOCKERM=""
  44 NR_OPS_VLMC=""
  45 NR_OPS_MAPPER=""
  46
  47 VERBOSITY_BLOCKERB=""
  48 VERBOSITY_BLOCKERM=""
  49 VERBOSITY_MAPPER=""
  50 VERBOSITY_VLMC=""
  51
  52
  53 #mt-pfiled specific options
  54 FILED_IMAGES=""
  55 FILED_MAPS=""
  56 PITHOS=""
  57 PITHOSMAPS=""
  58
  59 #mt-sosd specific options
  60 RADOS_POOL_MAPS=""
  61 RADOS_POOL_BLOCKS=""
  62
  63
  64
  65 def check_conf():
  66     def isExec(file_path):
  67         return os.path.isfile(file_path) and os.access(file_path, os.X_OK)
  68
  69     def validExec(program):
  70         for path in os.environ["PATH"].split(os.pathsep):
  71             exe_file = os.path.join(path, program)
  72             if isExec(exe_file):
  73                 return True
  74         return False
  75
  76
  77     def validPort(port, limit, name):
  78         try:
  79             if int(port) >= limit:
  80                 print str(port) + " >= " + limit
  81                 return False
  82         except:
  83             print "Invalid port "+name+" : " + str(port)
  84             return False
  85
  86         return True
  87
  88
  89     if not LOGS_PATH:
  90         print "LOGS_PATH is not set"
  91         return False
  92     if not PIDFILE_PATH:
  93         print "PIDFILE_PATH is not set"
  94         return False
  95
  96     try:
  97         if not os.path.isdir(str(LOGS_PATH)):
  98             print "LOGS_PATH "+str(LOGS_PATH)+" does not exist"
  99             return False
 100     except:
 101         print "LOGS_PATH doesn't exist or is not a directory"
 102         return False
 103
 104     try:
 105         os.makedirs(str(PIDFILE_PATH))
 106     except OSError as e:
 107         if e.errno == errno.EEXIST:
 108             if os.path.isdir(str(PIDFILE_PATH)):
 109                 pass
 110             else:
 111                 print str(PIDFILE_PATH) + " is not a directory"
 112                 return False
 113         else:
 114             print "Cannot create " + str(PIDFILE_PATH)
 115             return False
 116     except:
 117         print "PIDFILE_PATH is not set"
 118         return False
 119
 120     splitted_spec = str(SPEC).split(':')
 121     if len(splitted_spec) < 5:
 122         "Invalid spec"
 123         return False
 124
 125     xseg_type=splitted_spec[0]
 126     xseg_name=splitted_spec[1]
 127     xseg_ports=int(splitted_spec[2])
 128     xseg_heapsize=int(splitted_spec[3])
 129     xseg_align=int(splitted_spec[4])
 130
 131     if xseg_type != "segdev":
 132         print "Segment type not segdev"
 133         return False
 134     if xseg_name != "xsegbd":
 135         print "Segment name not equal xsegbd"
 136         return False
 137     if xseg_align != 12:
 138         print "Wrong alignemt"
 139         return False
 140
 141     for v in [VERBOSITY_BLOCKERB, VERBOSITY_BLOCKERM, VERBOSITY_MAPPER,
 142                     VERBOSITY_VLMC]:
 143          if v is None:
 144              print "Verbosity missing"
 145          try:
 146              if (int(v) > 3 or int(v) < 0):
 147                  print "Invalid verbosity " + str(v)
 148                  return False
 149          except:
 150              print "Invalid verbosity " + str(v)
 151              return False
 152
 153     for n in [NR_OPS_BLOCKERB, NR_OPS_BLOCKERM, NR_OPS_VLMC, NR_OPS_MAPPER]:
 154          if n is None:
 155              print "Nr ops missing"
 156          try:
 157              if (int(n) <= 0):
 158                  print "Invalid nr_ops " + str(n)
 159                  return False
 160          except:
 161              print "Invalid nr_ops " + str(n)
 162              return False
 163
 164     if not validPort(VTOOL, xseg_ports, "VTOOL"):
 165         return False
 166     if not validPort(MPORT, xseg_ports, "MPORT"):
 167         return False
 168     if not validPort(BPORT, xseg_ports, "BPORT"):
 169         return False
 170     if not validPort(MBPORT, xseg_ports, "MBPORT"):
 171         return False
 172     if not validPort(VPORT_START, xseg_ports, "VPORT_START"):
 173         return False
 174     if not validPort(VPORT_END, xseg_ports, "VPORT_END"):
 175         return False
 176
 177     global BLOCKER
 178     try:
 179         BLOCKER = available_storage[str(STORAGE)]
 180     except:
 181         print "Invalid storage " + str(STORAGE)
 182         print "Available storage: \"" + ', "'.join(available_storage) + "\""
 183         return False
 184
 185     if STORAGE=="files":
 186         if FILED_IMAGES and not os.path.isdir(str(FILED_IMAGES)):
 187              print "FILED_IMAGES invalid"
 188              return False
 189         if FILED_MAPS and not os.path.isdir(str(FILED_MAPS)):
 190              print "FILED_PATH invalid"
 191              return False
 192         if PITHOS and not os.path.isdir(str(PITHOS)):
 193              print "PITHOS invalid "
 194              return False
 195         if PITHOSMAPS and not os.path.isdir(str(PITHOSMAPS)):
 196              print "PITHOSMAPS invalid"
 197              return False
 198
 199     for p in [BLOCKER, MAPPER, VLMC]:
 200         if not validExec(p):
 201             print p + "is not a valid executable"
 202             return False
 203
 204     return True
 205
 206 def construct_peers():
 207     if BLOCKER == "pfiled":
 208         peer_blockerb = [BLOCKER,
 209                 ["-p" , str(BPORT), "-g", str(SPEC), "-n", str(NR_OPS_BLOCKERB),
 210                  str(PITHOS), str(FILED_IMAGES), "-d",
 211                 "-f", os.path.join(PIDFILE_PATH, "blockerb.pid")],
 212                 "blockerb"]
 213         peer_blockerm = [BLOCKER,
 214                 ["-p" , str(MBPORT), "-g", str(SPEC), "-n", str(NR_OPS_BLOCKERM),
 215                 str(PITHOSMAPS), str(FILED_MAPS), "-d",
 216                 "-f", os.path.join(PIDFILE_PATH, "blockerm.pid")],
 217                 "blockerm" ]
 218     elif BLOCKER == "mt-sosd":
 219         peer_blockerb = [BLOCKER,
 220                 ["-p" , str(BPORT), "-g", str(SPEC), "-n", str(NR_OPS_BLOCKERB),
 221                  "--pool", str(RADOS_POOL_BLOCKS), "-v", str(VERBOSITY_BLOCKERB),
 222                  "-d", "--pidfile", os.path.join(PIDFILE_PATH, "blockerb.pid"),
 223                  "-l", os.path.join(str(LOGS_PATH), "blockerb.log"),
 224                  "-t", "3"],
 225                  "blockerb"]
 226         peer_blockerm = [BLOCKER,
 227                 ["-p" , str(MBPORT), "-g", str(SPEC), "-n", str(NR_OPS_BLOCKERM),
 228                  "--pool", str(RADOS_POOL_MAPS), "-v", str(VERBOSITY_BLOCKERM),
 229                  "-d", "--pidfile", os.path.join(PIDFILE_PATH, "blockerm.pid"),
 230                  "-l", os.path.join(str(LOGS_PATH), "blockerm.log"),
 231                  "-t", "3"],
 232                  "blockerm"]
 233     elif BLOCKER == "mt-pfiled":
 234         peer_blockerb = [BLOCKER,
 235                 ["-p" , str(BPORT), "-g", str(SPEC), "-n", str(NR_OPS_BLOCKERB),
 236                  "--pithos", str(PITHOS), "--archip", str(FILED_IMAGES),
 237              "-v", str(VERBOSITY_BLOCKERB),
 238                  "-d", "--pidfile", os.path.join(PIDFILE_PATH, "blockerb.pid"),
 239                  "-l", os.path.join(str(LOGS_PATH), "blockerb.log"),
 240                  "-t", str(NR_OPS_BLOCKERB), "--prefix", "archip_"],
 241                  "blockerb"]
 242         peer_blockerm = [BLOCKER,
 243                 ["-p" , str(MBPORT), "-g", str(SPEC), "-n", str(NR_OPS_BLOCKERM),
 244                  "--pithos", str(PITHOSMAPS), "--archip", str(FILED_MAPS),
 245              "-v", str(VERBOSITY_BLOCKERM),
 246                  "-d", "--pidfile", os.path.join(PIDFILE_PATH, "blockerm.pid"),
 247                  "-l", os.path.join(str(LOGS_PATH), "blockerm.log"),
 248                  "-t", str(NR_OPS_BLOCKERM), "--prefix", "archip_"],
 249                  "blockerm"]
 250     else:
 251             sys.exit(-1)
 252
 253     peer_vlmcd = [VLMC,
 254              ["-t" , "1", "-sp",  str(VPORT_START), "-ep", str(VPORT_END),
 255               "-g", str(SPEC), "-n", str(NR_OPS_VLMC), "-bp", str(BPORT),
 256               "-mp", str(MPORT), "-d", "-v", str(VERBOSITY_VLMC),
 257               "--pidfile", os.path.join(PIDFILE_PATH, "vlmcd.pid"),
 258               "-l", os.path.join(str(LOGS_PATH), "vlmcd.log")
 259               ], "vlmcd"]
 260     peer_mapperd = [MAPPER,
 261              ["-t" , "1", "-p",  str(MPORT), "-mbp", str(MBPORT),
 262               "-g", str(SPEC), "-n", str(NR_OPS_MAPPER), "-bp", str(BPORT),
 263               "--pidfile", os.path.join(PIDFILE_PATH, "mapperd.pid"),
 264               "-v", str(VERBOSITY_MAPPER), "-d",
 265               "-l", os.path.join(str(LOGS_PATH), "mapperd.log")
 266               ], "mapperd"]
 267
 268     peers = []
 269     peers.append(peer_blockerb)
 270     peers.append(peer_blockerm)
 271     peers.append(peer_vlmcd)
 272     peers.append(peer_mapperd)
 273
 274     return peers
 275
 276
 277 def exclusive(fn):
 278     def exclusive_args(args):
 279         file = "/tmp/vlmc_map.lock"
 280         while True:
 281             try:
 282                 fd = os.open(file, os.O_CREAT|os.O_EXCL|os.O_WRONLY)
 283                 break;
 284             except Exception, reason:
 285                 print >> sys.stderr, reason
 286                 time.sleep(0.05)
 287         try:
 288             r = fn(args)
 289         finally:
 290             os.close(fd)
 291             os.unlink(file)
 292         return r
 293
 294     return exclusive_args
 295
 296 @exclusive
 297 def vlmc_showmapped(args):
 298     try:
 299         devices = os.listdir(os.path.join(XSEGBD_SYSFS, "devices/"))
 300     except:
 301         return -1
 302
 303     if not devices:
 304         return -1
 305     print "id\tpool\timage\tsnap\tdevice"
 306     try:
 307         for f in devices:
 308             d_id = open(XSEGBD_SYSFS + "devices/" + f + "/id").read().strip()
 309             target = open(XSEGBD_SYSFS + "devices/"+ f + "/target").read().strip()
 310
 311             print "%s\t%s\t%s\t%s\t%s" % (d_id, '-', target, '-', DEVICE_PREFIX +
 312             d_id)
 313     except Exception, reason:
 314         print >> sys.stderr, reason
 315         return -2
 316     return len(devices)
 317
 318 def loadrc(rc):
 319     try:
 320         if rc == None:
 321             execfile(os.path.expanduser(DEFAULTS), globals())
 322         else:
 323             execfile(rc, globals())
 324     except:
 325         print "Cannot read config file"
 326         sys.exit(1)
 327
 328     if not check_conf():
 329         sys.exit(1)
 330
 331 def loaded_modules():
 332     lines = open("/proc/modules").read().split("\n")
 333     modules = [f.split(" ")[0] for f in lines]
 334     return modules
 335
 336 def loaded_module(name):
 337     return name in loaded_modules()
 338
 339 def load_module(name):
 340     modules = loaded_modules()
 341     if name in modules:
 342         return 0
 343     cmd = ["modprobe -v %s" % name]
 344     try:
 345         check_call(cmd, shell=True);
 346     except Exception:
 347             sys.stderr.write("Module %s failed to load. \n" % name)
 348             return -1
 349     return 0
 350
 351 def unload_module(name):
 352     modules = loaded_modules()
 353     if name not in modules:
 354         return 0
 355     cmd = ["modprobe -rv %s" % name]
 356     print cmd
 357     try:
 358         check_call(cmd, shell=True);
 359     except Exception:
 360             sys.stderr.write("Module %s failed to unload. \n" % name)
 361             return -1
 362     return 0
 363
 364 def create_segment():
 365     #fixme blocking....
 366     cmd = ["xseg", str(SPEC), "create"]
 367     try:
 368         check_call(cmd, shell=False);
 369     except Exception:
 370             sys.stderr.write("Cannot create segment. \n")
 371             return -1
 372     return 0
 373
 374 def destroy_segment():
 375     #fixme blocking....
 376     cmd = ["xseg", str(SPEC), "destroy"]
 377     try:
 378         check_call(cmd, shell=False);
 379     except Exception:
 380             sys.stderr.write("Cannot destroy segment. \n")
 381             return 0
 382     return 0
 383
 384 def check_running(name, pid = -1):
 385     for p in psutil.process_iter():
 386         if p.name == name:
 387             if pid != -1:
 388                 if pid == p.pid:
 389                     return pid
 390             else:
 391                 return pid
 392     return -1
 393
 394 def check_pidfile(name):
 395     pidfile = os.path.join(PIDFILE_PATH, name + ".pid")
 396     pf = None
 397     try:
 398         pf = open(pidfile, "r")
 399         pid = int(pf.read())
 400         pf.close()
 401     except:
 402         if pf:
 403             pf.close()
 404         return -1
 405
 406     return pid
 407
 408 def start_peer(peer):
 409     cmd = [peer[0]] + peer[1]
 410     try:
 411         check_call(cmd, shell=False);
 412     except Exception:
 413         sys.stderr.write("Peer %s start failed.\n" % peer[0])
 414         return -1
 415     return 0
 416
 417 def stop_peer(peer):
 418     pid = check_pidfile(peer[2])
 419     if pid < 0:
 420         print " process not running"
 421         return -1
 422
 423     os.kill(pid, signal.SIGTERM)
 424     i = 0
 425     while check_running(peer[0], pid) > 0:
 426         time.sleep(0.1)
 427         i += 1
 428         if i > 150:
 429             print "process did not die in 15 secs"
 430             return -1
 431     return 0
 432
 433 def peer_running(peer):
 434     pid = check_pidfile(peer[2])
 435     if pid < 0:
 436         return -1
 437
 438     r = check_running(peer[0], pid)
 439     if r < 0:
 440         print "Peer " + peer[2] + " has valid pidfile but does not seem to be active"
 441     return 0
 442
 443
 444 def make_segdev():
 445     try:
 446         os.stat(str(CHARDEV_NAME))
 447         return -2
 448     except:
 449         pass
 450     cmd = ["mknod", str(CHARDEV_NAME), "c", str(CHARDEV_MAJOR), str(CHARDEV_MINOR)]
 451     print ' '.join(cmd)
 452     try:
 453         check_call(cmd, shell=False);
 454     except Exception:
 455         sys.stderr.write("Segdev device creation failed. \n")
 456         return -1
 457     return 0
 458
 459 def remove_segdev():
 460     try:
 461         os.stat(str(CHARDEV_NAME))
 462     except:
 463         return -2
 464     try:
 465         os.unlink(str(CHARDEV_NAME))
 466     except:
 467         sys.stderr.write("Segdev device removal failed. \n")
 468         return -1
 469
 470
 471 def start(args):
 472     if status(args) > 0:
 473         return -1
 474
 475     for m in modules:
 476         if load_module(m) < 0:
 477             stop(args)
 478             return -1
 479     time.sleep(0.5)
 480
 481     if make_segdev() < 0:
 482         stop(args)
 483         return -1
 484
 485     time.sleep(0.5)
 486
 487     if create_segment() < 0:
 488         stop(args)
 489         return -1
 490
 491     time.sleep(0.5)
 492
 493     for p in peers:
 494         if start_peer(p) < 0:
 495             stop(args)
 496             return -1
 497
 498
 499     if load_module(xsegbd) < 0:
 500         stop(args)
 501         return -1
 502     return 0
 503
 504 def stop(args):
 505     #check devices
 506     if vlmc_showmapped(args) > 0:
 507         print "Cannot stop archipelago. Mapped volumes exist"
 508         return -1
 509     if unload_module(xsegbd):
 510         return -1
 511     r = 0
 512     for p in reversed(peers):
 513         stop_peer(p)
 514
 515     remove_segdev()
 516
 517     for m in reversed(modules):
 518         unload_module(m)
 519     return 0
 520
 521 def status(args):
 522     r = 0
 523     if vlmc_showmapped(args) >= 0:
 524         r += 1
 525     if loaded_module(xsegbd):
 526         print "Xsegbd loaded"
 527         r += 1
 528     else:
 529         print "Xsegbd not loaded"
 530     for m in reversed(modules):
 531         if loaded_module(m):
 532             print m + " loaded"
 533             r += 1
 534         else:
 535             print m + " not loaded"
 536     for p in reversed(peers):
 537         if peer_running(p) < 0:
 538             print p[0] + " not running"
 539         else:
 540             print p[0] + " running"
 541             r += 1
 542     return r
 543
 544 def restart(args):
 545     r = stop(args)
 546     if r < 0:
 547         return r
 548     return start(args)
 549
 550 if __name__ == "__main__":
 551     # parse arguments and discpatch to the correct func
 552     parser = argparse.ArgumentParser(description='Archipelago tool')
 553     parser.add_argument('-c', '--config', type=str, nargs='?', help='config file')
 554     subparsers = parser.add_subparsers()
 555
 556     start_parser = subparsers.add_parser('start', help='Start archipelago')
 557     start_parser.set_defaults(func=start)
 558
 559     stop_parser = subparsers.add_parser('stop', help='Stop archipelago')
 560     stop_parser.set_defaults(func=stop)
 561
 562     status_parser = subparsers.add_parser('status', help='Archipelago status')
 563     status_parser.set_defaults(func=status)
 564
 565     restart_parser = subparsers.add_parser('restart', help='Restart archipelago')
 566     restart_parser.set_defaults(func=restart)
 567
 568     args = parser.parse_args()
 569     loadrc(args.config)
 570     peers = construct_peers()
 571     sys.exit(args.func(args))