The script we finally used to do the actual migration

[invirt/scripts/prod-migration.git] / xvm-migrate-machine

diff --git a/xvm-migrate-machine b/xvm-migrate-machine
old mode 100644 (file)
new mode 100755 (executable)
index 5dc8894..8d76225
--- a/xvm-migrate-machine
+++ b/xvm-migrate-machine
@@ -1,4 +1,4 @@
-#!/bin/python
+#!/usr/bin/python
 # Migrates the machine passed as arguments from the dev cluster.
 # To be run on the prod cluster.
 
@@ -6,6 +6,12 @@ from invirt import remctl as r
 from lib import database
 import subprocess
 import sys
+import time
+import os
+
+sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0)
+
+kinit = dict(principal='host/aperture-science.mit.edu', keytab='/etc/krb5.keytab')
 
 dev_db_uri = 'postgres://sipb-xen@sipb-xen-dev.mit.edu/sipb_xen'
 database.connect(dev_db_uri)
@@ -49,7 +55,7 @@ def take_data(machine_name):
   for r in disks + nics + [machine]:
     dev_sess.delete(r)
   
-  dev_sess.commit()
+  dev_sess.flush()
   
   for r in disks + nics + [machine]:
     dev_sess.expunge(r)
@@ -58,43 +64,87 @@ def take_data(machine_name):
   return machine
 
 ## add to prod db
-def restore_data(machine):
+def restore_data(machine, session):
   # The machine's type is still the one attached to the dev database;
   # get the right one
-  machine.type = prod_sess.query(database.Type).filter_by(type_id=machine.type.type_id).one()
-  prod_sess.begin()
-  prod_sess.save(machine)
-  prod_sess.commit()
+  machine.type = session.query(database.Type).filter_by(type_id=machine.type.type_id).one()
+  session.begin()
+  session.save(machine)
+  session.commit()
   
 def migrate_vm(machine_name):
   # Power off the VM on dev
   #
   # This has to be done first, because once the machine is deleted
   # from the database, we can't remctl for it anymore
-  out, err = r.remctl('xvm-remote.mit.edu', 'control', machine_name, 'destroy', err=True)
+  out, err = r.remctl('xvm-remote.mit.edu', 'control', machine_name, 'destroy', err=True, **kinit)
   print out
   
   machine = take_data(machine_name)
+  subprocess.call(['zwrite', '-d', '-c', 'xvm-auto', '-i', 'migration', '-s', 'XVM Migration Script', '-m', 
+                   'Migrating %s, disk size %0.2fG...' % (machine.name, sum(disk.size for disk in machine.disks) / 1024.0)])
   
+  success = True
   ## copy disk image... copy, copy...
   for disk in machine.disks:
     lvname='d_%s_%s' % (machine.name, disk.guest_device_name)
     
-    subprocess.check_call(['lvcreate', '-L%sM' % str(disk.size), '-n', lvname, 'xenvg'])
+    if 0 != subprocess.call(['lvcreate', '-L%sM' % str(disk.size), '-n', lvname, 'xenvg']):
+      success = False
+      break
     
-    ssh = subprocess.Popen(['ssh', '-o', 'GSSAPIDelegateCredentials=no',
-                'torchwood-institute.mit.edu',
+    ssh = subprocess.Popen(['rsh',
+                '10.6.0.165',
                 'dd', 'if=/dev/xenvg/%s' % lvname, 'bs=1M'],
                  stdout=subprocess.PIPE)
     dd = subprocess.Popen(['dd', 'of=/dev/xenvg/%s' % lvname, 'bs=1M'],
                 stdin=ssh.stdout)
-    dd.wait()
+    if 0 != dd.wait():
+      success = False
+      break
+    if 0 != ssh.wait():
+      success = False
+      break
+  
+  if not success:
+    dev_sess.rollback()
+    
+    print '==============================================='
+    print 'ERROR: VM %s failed to migrate' % machine.name
+    print '==============================================='
+  else:
+    restore_data(machine, prod_sess)
+    dev_sess.commit()
   
-  restore_data(machine)
+  return success
 
 if __name__ == '__main__':
-  for vm in sys.argv[1:]:
+  while True:
+    r.checkKinit(**kinit)
+    p = subprocess.Popen(['curl', '-s', '-k', '--negotiate', '-u', ':', 'https://xvm.mit.edu:442/offlist'], stdout=subprocess.PIPE)
+    if 0 != p.wait():
+        subprocess.call(['zwrite', '-d', '-c', 'xvm', '-i', 'migration', '-s', 'XVM Migration Script', '-m',
+                         'Failed to get list of remaining VMs. Will try again in 15 seconds'])
+        time.sleep(15)
+        continue
+    
+    next_line = p.stdout.read().split('\n')[0]
+    if next_line == '':
+      subprocess.call(['zwrite', '-d', '-c', 'xvm', '-i', 'migration', '-s', 'XVM Migration Script', '-m',
+                       'XVM migration complete'])
+      break
+    
+    next, uptime = next_line.split('\t')[:2]
+    
     print '==============================================='
-    print 'Migrating %s' % vm
+    print 'Migrating %s' % next
     print '==============================================='
-    migrate_vm(vm.strip())
+    if not migrate_vm(next):
+      subprocess.call(['zwrite', '-d', '-c', 'xvm', '-i', 'migration', '-s', 'XVM Migration Script', '-m',
+                       'Error in migrating %s' % next])
+      sys.exit(1)
+    if uptime.strip() != '':
+      r.remctl('xvm-remote-dev.mit.edu', 'control', next, 'create', **kinit)
+    
+    subprocess.call(['zwrite', '-d', '-c', 'xvm-auto', '-i', 'migration', '-s', 'XVM Migration Script', '-m',
+                     'done'])