Balance LVM requests (LP: #307361)

* Add "monocast" method to send remctl requests to exactly one
   randomly-selected host.
 * Send LVM requests to randomly-selected host for redundancy and
   load-balancing (LP: #307361).

svn path=/trunk/packages/invirt-remote/; revision=2495

diff --git a/debian/changelog b/debian/changelog
index 09a0af0..174bcf0 100644 (file)
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,12 @@
+invirt-remote (0.4.2) unstable; urgency=low
+
+  * Add "monocast" method to send remctl requests to exactly one 
+    randomly-selected host.
+  * Send LVM requests to randomly-selected host for redundancy and
+    load-balancing (LP: #307361).
+
+ -- Alex Dehnert <adehnert@mit.edu>  Wed, 07 Oct 2009 03:19:30 -0400
+
 invirt-remote (0.4.1) unstable; urgency=low
 
   [Paul Weaver]

diff --git a/python/remote/__init__.py b/python/remote/__init__.py
index adaa9b7..201bf84 100644 (file)
--- a/python/remote/__init__.py
+++ b/python/remote/__init__.py
@@ -1 +1,2 @@
 from bcast import bcast
+from monocast import monocast

diff --git a/python/remote/monocast.py b/python/remote/monocast.py
new file mode 100644 (file)
index 0000000..1b39b57
--- /dev/null
+++ b/python/remote/monocast.py
@@ -0,0 +1,34 @@
+from subprocess import PIPE, Popen
+from invirt.config import structs as config
+import random
+
+hostnames = [ h.hostname for h in config.hosts ]
+
+def monocast(args, hosts = hostnames, randomize = True):
+    """
+    Given a command and a list of hostnames or IPs, issue the command to each
+    node until it connects to one of the nodes.
+
+    Returns:
+        host the command ran on
+        hosts that could not be contacted
+        returncode of remctl
+        stdout of remctl
+        stderr of remctl
+    """
+    if(randomize):
+        hosts = random.sample(hosts, len(hosts))
+    hostsdown = []
+    for host in hosts:
+        pipe = Popen(['remctl', host, 'remote', 'web'] + args, stdout=PIPE, stderr=PIPE)
+        output = pipe.communicate()
+        if pipe.returncode != 0:
+            if output[1].startswith('remctl: cannot connect to %s' % host):
+                hostsdown.append(host)
+            else:
+                #raise RuntimeError("remctl to host %s returned non-zero exit status %d; stderr:\n%s"
+                #                   % (host, pipe.returncode, output[1]))
+                return (host, hostsdown, pipe.returncode,) + output
+        else:
+            return (host, hostsdown, pipe.returncode,) + output
+    raise RuntimeError("Failed to contact any hosts: tried %s" % (hostsdown, ))

diff --git a/server/usr/sbin/invirt-remote-lvm b/server/usr/sbin/invirt-remote-lvm
new file mode 100755 (executable)
index 0000000..1c69107
--- /dev/null
+++ b/server/usr/sbin/invirt-remote-lvm
@@ -0,0 +1,24 @@
+#!/usr/bin/python
+
+"""
+Run an LVM command on one host, with failover for downed hosts.
+"""
+
+from invirt.remote import monocast
+import sys
+import os
+
+def main(argv):
+    # Query each of the hosts.
+    if(os.path.exists("/etc/invirt/nolvm")):
+        sys.stderr.write("LVM operations are temporarily disabled for maintenance, sorry.\n")
+        return 2
+    result = monocast(argv[1:])
+    #print "Hostname: %s; down=%s" % (result[0], result[1])
+    sys.stdout.write(result[3]) # stdout
+    sys.stderr.write(result[4]) # stderr
+
+if __name__ == '__main__':
+    sys.exit(main(sys.argv))
+
+# vim:et:sw=4:ts=4

diff --git a/server/usr/sbin/invirt-remote-proxy b/server/usr/sbin/invirt-remote-proxy
index 7608ad3..e92e692 100755 (executable)
--- a/server/usr/sbin/invirt-remote-proxy
+++ b/server/usr/sbin/invirt-remote-proxy
@@ -1,29 +1,26 @@
-#!/bin/bash
-# invoke as invirt-remote-proxy-$TYPE, with "TYPE" in the remctl sense.
+#!/bin/bash                                                                                         
+# invoke as invirt-remote-proxy-$TYPE, with "TYPE" in the remctl sense.                             
 
 klist -s || kinit -k
 
 TYPE="${0##*-}"
 case "$TYPE" in
     control )
-       MACHINE="$1"; SERVICE="$2"; shift; shift ;;
+        MACHINE="$1"; SERVICE="$2"; shift; shift ;;
     * )
-       SERVICE="$1"; shift ;;
+        SERVICE="$1"; shift ;;
 esac
 
 case "$TYPE/$SERVICE" in
     web/listvms )
-       invirt-remote-listvms "$@" ;;
+        invirt-remote-listvms "$@" ;;
     web/vnccert )
         invirt-remote-vnccert "$@" ;;
     web/availability | web/avail )
         invirt-remote-availability "$@" ;;
     web/lvcreate | web/lvremove | web/lvrename | web/lvresize )
-        if [ -f "/etc/invirt/nolvm" ]; then
-            echo "LVM operations are temporarily disabled for maintenance, sorry." >&2
-            exit 2
-        fi
-        remctl "$(invirt-getconf hosts.0.hostname)" remote "$TYPE" "$SERVICE" "$@" ;;
+        invirt-remote-lvm "$SERVICE" "$@"
+        ;;
     control/help )
         invirt-remctl-help ;;
     control/create|control/install )
@@ -31,14 +28,14 @@ case "$TYPE/$SERVICE" in
             echo "Booting VMs is temporarily disabled for maintenance, sorry." >&2
             exit 2
         fi
-       invirt-remote-create "$SERVICE" "$MACHINE" "$@" ;;
+        invirt-remote-create "$SERVICE" "$MACHINE" "$@" ;;
     control/listhost|control/list-host )
-       invirt-remote-listhost "$MACHINE" "$@" ;;
+        invirt-remote-listhost "$MACHINE" "$@" ;;
     control/* )
-       # Everything but create must go where the VM is already running.
-       invirt-remote-control "$MACHINE" "$SERVICE" "$@" ;;
+        # Everything but create must go where the VM is already running.
+        invirt-remote-control "$MACHINE" "$SERVICE" "$@" ;;
     * )
-        echo "ERROR: invalid subcommand"
+        echo "ERROR: invalid subcommand $TYPE/$SERVICE"
         exit 34
         ;;
 esac