The script we finally used to do the actual migration

[invirt/scripts/prod-migration.git] / xvm-migrate-machine

diff --git a/xvm-migrate-machine b/xvm-migrate-machine
index efc3416..8d76225 100755 (executable)
--- a/xvm-migrate-machine
+++ b/xvm-migrate-machine
@@ -6,6 +6,12 @@ from invirt import remctl as r
 from lib import database
 import subprocess
 import sys
 from lib import database
 import subprocess
 import sys

+import time
+import os
+
+sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0)
+
+kinit = dict(principal='host/aperture-science.mit.edu', keytab='/etc/krb5.keytab')

 
 dev_db_uri = 'postgres://sipb-xen@sipb-xen-dev.mit.edu/sipb_xen'
 database.connect(dev_db_uri)
 
 dev_db_uri = 'postgres://sipb-xen@sipb-xen-dev.mit.edu/sipb_xen'
 database.connect(dev_db_uri)

@@ -49,7 +55,7 @@ def take_data(machine_name):
   for r in disks + nics + [machine]:
     dev_sess.delete(r)
   
   for r in disks + nics + [machine]:
     dev_sess.delete(r)
   

-  dev_sess.commit()
+  dev_sess.flush()

   
   for r in disks + nics + [machine]:
     dev_sess.expunge(r)
   
   for r in disks + nics + [machine]:
     dev_sess.expunge(r)

@@ -71,10 +77,12 @@ def migrate_vm(machine_name):
   #
   # This has to be done first, because once the machine is deleted
   # from the database, we can't remctl for it anymore
   #
   # This has to be done first, because once the machine is deleted
   # from the database, we can't remctl for it anymore

-  out, err = r.remctl('xvm-remote.mit.edu', 'control', machine_name, 'destroy', err=True)
+  out, err = r.remctl('xvm-remote.mit.edu', 'control', machine_name, 'destroy', err=True, **kinit)

   print out
   
   machine = take_data(machine_name)
   print out
   
   machine = take_data(machine_name)

+  subprocess.call(['zwrite', '-d', '-c', 'xvm-auto', '-i', 'migration', '-s', 'XVM Migration Script', '-m', 
+                   'Migrating %s, disk size %0.2fG...' % (machine.name, sum(disk.size for disk in machine.disks) / 1024.0)])

   
   success = True
   ## copy disk image... copy, copy...
   
   success = True
   ## copy disk image... copy, copy...

@@ -85,8 +93,8 @@ def migrate_vm(machine_name):
       success = False
       break
     
       success = False
       break
     

-    ssh = subprocess.Popen(['ssh', '-o', 'GSSAPIDelegateCredentials=no',
-                'torchwood-institute.mit.edu',
+    ssh = subprocess.Popen(['rsh',
+                '10.6.0.165',

                 'dd', 'if=/dev/xenvg/%s' % lvname, 'bs=1M'],
                  stdout=subprocess.PIPE)
     dd = subprocess.Popen(['dd', 'of=/dev/xenvg/%s' % lvname, 'bs=1M'],
                 'dd', 'if=/dev/xenvg/%s' % lvname, 'bs=1M'],
                  stdout=subprocess.PIPE)
     dd = subprocess.Popen(['dd', 'of=/dev/xenvg/%s' % lvname, 'bs=1M'],

@@ -99,17 +107,44 @@ def migrate_vm(machine_name):
       break
   
   if not success:
       break
   
   if not success:

-    restore_data(machine, dev_sess)
+    dev_sess.rollback()

     
     print '==============================================='
     print 'ERROR: VM %s failed to migrate' % machine.name
     print '==============================================='
   else:
     restore_data(machine, prod_sess)
     
     print '==============================================='
     print 'ERROR: VM %s failed to migrate' % machine.name
     print '==============================================='
   else:
     restore_data(machine, prod_sess)

+    dev_sess.commit()
+  
+  return success

 
 if __name__ == '__main__':
 
 if __name__ == '__main__':

-  for vm in sys.argv[1:]:
+  while True:
+    r.checkKinit(**kinit)
+    p = subprocess.Popen(['curl', '-s', '-k', '--negotiate', '-u', ':', 'https://xvm.mit.edu:442/offlist'], stdout=subprocess.PIPE)
+    if 0 != p.wait():
+        subprocess.call(['zwrite', '-d', '-c', 'xvm', '-i', 'migration', '-s', 'XVM Migration Script', '-m',
+                         'Failed to get list of remaining VMs. Will try again in 15 seconds'])
+        time.sleep(15)
+        continue
+    
+    next_line = p.stdout.read().split('\n')[0]
+    if next_line == '':
+      subprocess.call(['zwrite', '-d', '-c', 'xvm', '-i', 'migration', '-s', 'XVM Migration Script', '-m',
+                       'XVM migration complete'])
+      break
+    
+    next, uptime = next_line.split('\t')[:2]
+    

     print '==============================================='
     print '==============================================='

-    print 'Migrating %s' % vm
+    print 'Migrating %s' % next

     print '==============================================='
     print '==============================================='

-    migrate_vm(vm.strip())
+    if not migrate_vm(next):
+      subprocess.call(['zwrite', '-d', '-c', 'xvm', '-i', 'migration', '-s', 'XVM Migration Script', '-m',
+                       'Error in migrating %s' % next])
+      sys.exit(1)
+    if uptime.strip() != '':
+      r.remctl('xvm-remote-dev.mit.edu', 'control', next, 'create', **kinit)
+    
+    subprocess.call(['zwrite', '-d', '-c', 'xvm-auto', '-i', 'migration', '-s', 'XVM Migration Script', '-m',
+                     'done'])