The script we finally used to do the actual migration

[invirt/scripts/prod-migration.git] / xvm-migrate-machine

diff --git a/xvm-migrate-machine b/xvm-migrate-machine
old mode 100644 (file)
new mode 100755 (executable)
index d8c729e..8d76225
--- a/xvm-migrate-machine
+++ b/xvm-migrate-machine
@@ -1,30 +1,150 @@
-##!/bin/bash
-# Migrates the machine named $1 from the dev cluster.
+#!/usr/bin/python
+# Migrates the machine passed as arguments from the dev cluster.

 # To be run on the prod cluster.
 
 # To be run on the prod cluster.
 

-## The present version is NOT A REAL SCRIPT.
-## Things may not even be tested.  Copy and paste.
+from invirt import remctl as r
+from lib import database
+import subprocess
+import sys
+import time
+import os

 
 

-## dump from dev db; save info well
+sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0)

 
 

-#echo "\\a \\t \\\\ select * from machines where name = '$MACHINE';" \
-# | psql -h xvm -U sipb-xen sipb_xen -q
-## 581|fsck|256|price|price|2ab6638f-3f65-2b32-3fd3-c16b74a9b7fe|linux|f|1|price|test|f
-## watch out for funny characters in description; better (non-)quoting needed
-## also disks, nics
+kinit = dict(principal='host/aperture-science.mit.edu', keytab='/etc/krb5.keytab')

 
 

-## remove from dev db; ideally atomic with dump
+dev_db_uri = 'postgres://sipb-xen@sipb-xen-dev.mit.edu/sipb_xen'
+database.connect(dev_db_uri)
+dev_sess = database.session

 
 

-## shut down if up
-#remctl remote control $MACHINE destroy
+database.connect()
+prod_sess = database.session

 
 

-## copy disk image... copy, copy...
-## for each disk:
-lvname="d_${MACHINE}_${guest_device_name}"
-lvcreate xenvg -n "$lvname" -L "${size}"M
-ssh t-i dd if=/dev/xenvg/"$lvname" of=/dev/stdout bs=1M \
- | dd if=/dev/stdin of=/dev/xenvg/"$lvname" bs=1M
+## dump from dev db
+def take_data(machine_name):
+  dev_sess.begin()
+  machine = dev_sess.query(database.Machine).filter_by(name=machine_name).one()
+  
+  # Clean out the ACL just so we don't have to think about it
+  machine.acl = []
+  dev_sess.update(machine)
+  
+  print 'VM Info:'
+  print '  name: %s' % machine.name
+  print '  description: %s' % machine.description
+  print '  cpus: %s' % machine.cpus
+  print '  memory: %s' % machine.memory
+  print '  owner: %s' % machine.owner
+  print '  contact: %s' % machine.contact
+  print '  administrator: %s' % machine.administrator
+  print '  uuid: %s' % machine.uuid
+  print '  type: %s' % machine.type.type_id
+  print '  autorestart: %s' % machine.autorestart
+  print '  adminable: %s' % machine.adminable
+  print '  Disks:'
+  for disk in machine.disks:
+    print '  - %s (%s)' % (disk.guest_device_name, disk.size)
+  print '  NICs:'
+  for nic in machine.nics:
+    print '  - %s, %s, %s' % (nic.mac_addr, nic.ip, nic.hostname)
+  print '==============================================='
+  print
+  
+  disks = machine.disks
+  nics = machine.nics
+  for r in disks + nics + [machine]:
+    dev_sess.delete(r)
+  
+  dev_sess.flush()
+  
+  for r in disks + nics + [machine]:
+    dev_sess.expunge(r)
+    del r._instance_key
+  
+  return machine

 
 

-## add to dev db
+## add to prod db
+def restore_data(machine, session):
+  # The machine's type is still the one attached to the dev database;
+  # get the right one
+  machine.type = session.query(database.Type).filter_by(type_id=machine.type.type_id).one()
+  session.begin()
+  session.save(machine)
+  session.commit()
+  
+def migrate_vm(machine_name):
+  # Power off the VM on dev
+  #
+  # This has to be done first, because once the machine is deleted
+  # from the database, we can't remctl for it anymore
+  out, err = r.remctl('xvm-remote.mit.edu', 'control', machine_name, 'destroy', err=True, **kinit)
+  print out
+  
+  machine = take_data(machine_name)
+  subprocess.call(['zwrite', '-d', '-c', 'xvm-auto', '-i', 'migration', '-s', 'XVM Migration Script', '-m', 
+                   'Migrating %s, disk size %0.2fG...' % (machine.name, sum(disk.size for disk in machine.disks) / 1024.0)])
+  
+  success = True
+  ## copy disk image... copy, copy...
+  for disk in machine.disks:
+    lvname='d_%s_%s' % (machine.name, disk.guest_device_name)
+    
+    if 0 != subprocess.call(['lvcreate', '-L%sM' % str(disk.size), '-n', lvname, 'xenvg']):
+      success = False
+      break
+    
+    ssh = subprocess.Popen(['rsh',
+                '10.6.0.165',
+                'dd', 'if=/dev/xenvg/%s' % lvname, 'bs=1M'],
+                 stdout=subprocess.PIPE)
+    dd = subprocess.Popen(['dd', 'of=/dev/xenvg/%s' % lvname, 'bs=1M'],
+                stdin=ssh.stdout)
+    if 0 != dd.wait():
+      success = False
+      break
+    if 0 != ssh.wait():
+      success = False
+      break
+  
+  if not success:
+    dev_sess.rollback()
+    
+    print '==============================================='
+    print 'ERROR: VM %s failed to migrate' % machine.name
+    print '==============================================='
+  else:
+    restore_data(machine, prod_sess)
+    dev_sess.commit()
+  
+  return success

 
 

-## power on if desired
+if __name__ == '__main__':
+  while True:
+    r.checkKinit(**kinit)
+    p = subprocess.Popen(['curl', '-s', '-k', '--negotiate', '-u', ':', 'https://xvm.mit.edu:442/offlist'], stdout=subprocess.PIPE)
+    if 0 != p.wait():
+        subprocess.call(['zwrite', '-d', '-c', 'xvm', '-i', 'migration', '-s', 'XVM Migration Script', '-m',
+                         'Failed to get list of remaining VMs. Will try again in 15 seconds'])
+        time.sleep(15)
+        continue
+    
+    next_line = p.stdout.read().split('\n')[0]
+    if next_line == '':
+      subprocess.call(['zwrite', '-d', '-c', 'xvm', '-i', 'migration', '-s', 'XVM Migration Script', '-m',
+                       'XVM migration complete'])
+      break
+    
+    next, uptime = next_line.split('\t')[:2]
+    
+    print '==============================================='
+    print 'Migrating %s' % next
+    print '==============================================='
+    if not migrate_vm(next):
+      subprocess.call(['zwrite', '-d', '-c', 'xvm', '-i', 'migration', '-s', 'XVM Migration Script', '-m',
+                       'Error in migrating %s' % next])
+      sys.exit(1)
+    if uptime.strip() != '':
+      r.remctl('xvm-remote-dev.mit.edu', 'control', next, 'create', **kinit)
+    
+    subprocess.call(['zwrite', '-d', '-c', 'xvm-auto', '-i', 'migration', '-s', 'XVM Migration Script', '-m',
+                     'done'])