From: Evan Broder Date: Sun, 7 Dec 2008 23:53:01 +0000 (-0500) Subject: Merge invirt-remote-host and invirt-remote-server into invirt-remote X-Git-Tag: 0.2.0^0 X-Git-Url: http://xvm.mit.edu/gitweb/invirt/packages/invirt-remote.git/commitdiff_plain/ccdd1009d00397d0f81fa9a95aa96faf448b9e94?hp=437a816b13b6701cdafc6fb7363324bd60a7efbe Merge invirt-remote-host and invirt-remote-server into invirt-remote svn path=/trunk/packages/invirt-remote/; revision=1822 --- diff --git a/debian/changelog b/debian/changelog index 209bb16..d0502bc 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,248 +1,267 @@ -invirt-remote-host (0.0.9) unstable; urgency=low +invirt-remote (0.2.0) unstable; urgency=low - * Use `xm reboot` for reboots instead of a destroy and create + * Merge invirt-remote-server and invirt-remote-host into invirt-remote + (LP: #305682) - -- Evan Broder Thu, 20 Nov 2008 10:54:13 -0500 + -- Evan Broder Sun, 07 Dec 2008 18:33:59 -0500 -invirt-remote-host (0.0.8) unstable; urgency=low +invirt-remote-server (0.1.6) unstable; urgency=low - * I know this is the wrong answer, but doing lvchange -a n twice before - deleting an LV tends to make it less likely to error out + * Fix a gen_config -> gen_files - -- Evan Broder Tue, 18 Nov 2008 11:55:22 -0500 + -- Greg Price Sat, 22 Nov 2008 19:27:36 -0500 -invirt-remote-host (0.0.7) unstable; urgency=low +invirt-remote-server (0.1.5) unstable; urgency=low - * Actually remove the unused vmboot remctl + * Fix a typo in calling /lib/init/gen-files.sh - -- Evan Broder Wed, 12 Nov 2008 18:27:48 -0500 + -- Evan Broder Sat, 22 Nov 2008 05:43:06 -0500 -invirt-remote-host (0.0.6) unstable; urgency=low +invirt-remote-server (0.1.4) unstable; urgency=low - * Use invoke-rc.d instead of calling init scripts directly + * Steal some init script ideas from debathena-pyhesiodfs - -- Evan Broder Fri, 31 Oct 2008 06:27:38 -0400 + -- Evan Broder Sat, 22 Nov 2008 05:31:49 -0500 -invirt-remote-host (0.0.5) unstable; urgency=low +invirt-remote-server (0.1.3) unstable; urgency=low - * Add a `remote web vnccert` to print out the VNC certificate + * Switched to using "raw" select expressions instead of using the ORM; + allows the same speed as the old object-based system but without any + caching necessary. - -- Evan Broder Tue, 28 Oct 2008 21:33:28 -0400 + -- Quentin Smith Fri, 21 Nov 2008 04:15:50 -0500 -invirt-remote-host (0.0.4) unstable; urgency=low +invirt-remote-server (0.1.2) unstable; urgency=low - * Fix a typo in invirt-vmcontrol + * Update remctl help with new reboot functionality - -- Evan Broder Tue, 28 Oct 2008 20:11:44 -0400 + -- Evan Broder Thu, 20 Nov 2008 10:57:39 -0500 -invirt-remote-host (0.0.3) unstable; urgency=low +invirt-remote-server (0.1.1) unstable; urgency=low - * Add support for a `remote control vnctoken` remctl for generating a - VNC auth token + * Now that we're doing real caching, cache for less time - -- Evan Broder Tue, 28 Oct 2008 19:47:22 -0400 + -- Evan Broder Mon, 17 Nov 2008 13:18:23 -0500 -invirt-remote-host (0.0.2) unstable; urgency=low +invirt-remote-server (0.1.0) unstable; urgency=low - * sipb-database -> invirt-database + * Add real caching to remconffs - -- Evan Broder Sat, 25 Oct 2008 18:05:17 -0400 + -- Evan Broder Mon, 17 Nov 2008 13:05:32 -0500 -invirt-remote-host (0.0.1) unstable; urgency=low +invirt-remote-server (0.0.18) unstable; urgency=low - * sipb-xen-remctl-auto -> invirt-remote-host + * Add a "help" command for the control remctl - -- Evan Broder Fri, 24 Oct 2008 21:54:35 -0400 + -- Evan Broder Sat, 15 Nov 2008 19:02:19 -0500 -sipb-xen-remctl-auto (1.5.2) unstable; urgency=low +invirt-remote-server (0.0.17) unstable; urgency=low - * sipb-xen-console-server -> invirt-console-host + * Get the host with the most RAM based on the maximum recoverable + memory, not the memory currently available to Xen - -- Greg Price Fri, 24 Oct 2008 01:43:15 -0400 + -- Evan Broder Sat, 15 Nov 2008 10:30:10 -0500 -sipb-xen-remctl-auto (1.5.1) unstable; urgency=low +invirt-remote-server (0.0.16) unstable; urgency=low - * Return the new invalid-command error code in sipb-xen-vmcontrol + * When returning from a bcast, put together the stdout, not the stderr - -- Evan Broder Wed, 22 Oct 2008 00:51:34 -0400 + -- Evan Broder Mon, 10 Nov 2008 16:19:33 -0500 -sipb-xen-remctl-auto (1.5) unstable; urgency=low +invirt-remote-server (0.0.15) unstable; urgency=low - * Distinguish between Xen errors and invalid commands by returning a - completely outlandish error code that's not likely to conflict with - anything + * Include invirt.remote Python module + * Ignore erroneous responses from a server if it's because the server is + down - -- Evan Broder Wed, 22 Oct 2008 00:47:50 -0400 + -- Evan Broder Mon, 10 Nov 2008 15:19:28 -0500 -sipb-xen-remctl-auto (1.4) unstable; urgency=low +invirt-remote-server (0.0.14) unstable; urgency=low - * Correctly extract the CDROM image from the remctl arguments + * Don't depend on invirt-mail-config - -- Evan Broder Sat, 18 Oct 2008 17:19:57 -0400 + -- Evan Broder Thu, 06 Nov 2008 22:51:19 -0500 -sipb-xen-remctl-auto (1.3.1) unstable; urgency=low +invirt-remote-server (0.0.13) unstable; urgency=low - * Quote the installer arguments correctly + * Depend on invirt-mail-config - -- Evan Broder Fri, 10 Oct 2008 01:47:54 -0400 + -- Evan Broder Thu, 06 Nov 2008 21:49:27 -0500 -sipb-xen-remctl-auto (1.3) unstable; urgency=low +invirt-remote-server (0.0.12) unstable; urgency=low - * Respond to remctl install requests by passing the options onto Xen + * shorten initscript with std-init, correct the usage message - -- Evan Broder Thu, 09 Oct 2008 01:57:30 -0400 + -- Greg Price Wed, 05 Nov 2008 19:59:18 -0500 -sipb-xen-remctl-auto (1.2) unstable; urgency=low +invirt-remote-server (0.0.11) unstable; urgency=low - * Exit successfully if inetd is already running; package fails to - install otherwise + * Don't proxy requests for the VNC cert if the requested hostname isn't + an Invirt host - -- Evan Broder Mon, 06 Oct 2008 05:03:18 -0400 + -- Evan Broder Tue, 28 Oct 2008 23:17:10 -0400 -sipb-xen-remctl-auto (1.1) unstable; urgency=low +invirt-remote-server (0.0.10) unstable; urgency=low - * Explicitly lock a volume before deleting it + * All remctls of type 'web' have the same ACL - glob them together - -- Evan Broder Sun, 05 Oct 2008 18:01:34 -0400 + -- Evan Broder Tue, 28 Oct 2008 23:05:42 -0400 -sipb-xen-remctl-auto (1.0.21) unstable; urgency=low +invirt-remote-server (0.0.9) unstable; urgency=low - * start inetd too in init script + * Add support for a `web vnccert` remctl call - -- Greg Price Wed, 01 Oct 2008 20:43:29 -0400 + -- Evan Broder Tue, 28 Oct 2008 22:45:29 -0400 -sipb-xen-remctl-auto (1.0.20) unstable; urgency=low +invirt-remote-server (0.0.8) unstable; urgency=low - * generate config files using mako + * sipb-xen-base -> invirt-base - -- Yang Zhang Thu, 14 Aug 2008 15:18:46 -0400 + -- Evan Broder Tue, 28 Oct 2008 04:23:32 -0400 -sipb-xen-remctl-auto (1.0.19) unstable; urgency=low +invirt-remote-server (0.0.7) unstable; urgency=low - * simplify the init script a la sipb-xen-dev + * sipb-xen-database-common -> invirt-database - -- Yang Zhang Thu, 7 Aug 2008 22:09:28 -0400 + -- Evan Broder Sat, 25 Oct 2008 21:03:13 -0400 -sipb-xen-remctl-auto (1.0.18) unstable; urgency=low +invirt-remote-server (0.0.6) unstable; urgency=low - * use invirt config in sipb-xen-lvm - * added .init script to generate /etc/remctl/acl/remote - * sipb_xen_database -> invirt.database + * Remove dependency on sipb-xen-chrony-config - we need to take care of + the clock, but chrony isn't the right answer - -- Yang Zhang Sun, 3 Aug 2008 00:46:28 -0400 + -- Evan Broder Sat, 25 Oct 2008 19:16:08 -0400 -sipb-xen-remctl-auto (1.0.17) unstable; urgency=low +invirt-remote-server (0.0.5) unstable; urgency=low - * catch another version of the xenstore error in the listvms race + * Rename a remaining sipb-xen file to invirt - -- Greg Price Wed, 23 Jul 2008 01:12:01 -0400 + -- Evan Broder Fri, 24 Oct 2008 22:21:30 -0400 -sipb-xen-remctl-auto (1.0.16) unstable; urgency=low +invirt-remote-server (0.0.4) unstable; urgency=low - * support YAML as well as JSON (broder, quentin, andersk, long ago) - * fix a race in listvms when a machine is shutting down + * Kill DEB_AUTO_UPDATE_DEBIAN_CONTROL - -- Greg Price Tue, 22 Jul 2008 23:46:49 -0400 + -- Evan Broder Fri, 24 Oct 2008 13:49:45 -0400 -sipb-xen-remctl-auto (1.0.15) unstable; urgency=low +invirt-remote-server (0.0.3) unstable; urgency=low - * Compute time differences on server to avoid drift + * Standardize on "Invirt project" - -- Quentin Smith Thu, 15 May 2008 20:58:04 -0400 + -- Evan Broder Fri, 24 Oct 2008 13:33:25 -0400 -sipb-xen-remctl-auto (1.0.14) unstable; urgency=low +invirt-remote-server (0.0.2) unstable; urgency=low - * Use more efficient python API to get VM information, and return it - in a Python dict structure. + * oops, update control from control.in + * configurize web acl - -- Quentin Smith Thu, 15 May 2008 20:29:03 -0400 + -- Greg Price Fri, 24 Oct 2008 00:28:08 -0400 -sipb-xen-remctl-auto (1.0.13) unstable; urgency=low +invirt-remote-server (0.0.1) unstable; urgency=low - * Switch to just using the remote server. + * sipb-xen -> invirt - -- Greg Price Sat, 10 May 2008 21:26:42 -0400 + -- Greg Price Thu, 23 Oct 2008 23:58:36 -0400 -sipb-xen-remctl-auto (1.0.12) unstable; urgency=low +sipb-xen-remote-server (0.10.2) unstable; urgency=low - * Remove an obsolete line from the remctl config. + * There's a race condition for if the VM gets powered off between the + listvms and the actual request. Solve the problem correctly by using a + non-conflicting error code to indicate an invalid command - -- Greg Price Sun, 4 May 2008 22:36:07 -0400 + -- Evan Broder Wed, 22 Oct 2008 00:46:38 -0400 -sipb-xen-remctl-auto (1.0.11) unstable; urgency=low +sipb-xen-remote-server (0.10.1) unstable; urgency=low - * Close file descriptors for update-conserver on destroy/shutdown. + * Make both list-host and listhost valid commands, and for parity with + list-long, document list-host - -- Eric Price Mon, 21 Apr 2008 21:28:12 -0400 + -- Evan Broder Wed, 22 Oct 2008 00:36:32 -0400 -sipb-xen-remctl-auto (1.0.10) unstable; urgency=low +sipb-xen-remote-server (0.10) unstable; urgency=low - * Delay running sipb-xen-update-conserver + * If a command is invalid, but the machine's on, give an actually useful + error message - -- Evan Broder Mon, 21 Apr 2008 20:26:28 -0400 + -- Evan Broder Wed, 22 Oct 2008 00:29:50 -0400 -sipb-xen-remctl-auto (1.0.9) unstable; urgency=low +sipb-xen-remote-server (0.9.2) unstable; urgency=low - * Change the way that conserver is updated + * Expose an interface to the autoinstaller over remctl + * Validate options to be passed to the autoinstaller on the remctl + server - -- Evan Broder Wed, 2 Apr 2008 04:36:04 -0400 + -- Evan Broder Thu, 09 Oct 2008 02:06:00 -0400 -sipb-xen-remctl-auto (1.0.8) unstable; urgency=low +sipb-xen-remote-server (0.9.1) unstable; urgency=low - * Add support for updating conserver when VMs are turned on and off + * Fix a bug in the database handling code - -- Evan Broder Wed, 2 Apr 2008 00:40:23 -0400 + -- Evan Broder Fri, 03 Oct 2008 18:42:48 -0400 -sipb-xen-remctl-auto (1.0.7) unstable; urgency=low +sipb-xen-remote-server (0.9) unstable; urgency=low - * changes for installer system + [Evan Broder] + * Make the acl/ subdir of RemConfFS listable + + [Greg Price] + * Update RemConfFS for new SQLAlchemy code - -- Tim Abbott Sun, 28 Oct 2007 22:47:59 -0400 + -- Evan Broder Wed, 01 Oct 2008 19:28:23 -0400 -sipb-xen-remctl-auto (1.0.6) unstable; urgency=low +sipb-xen-remote-server (0.8) unstable; urgency=low - * get rid of old "vmboot" call - * make "create" not destroy running machines. + * Use RouteFS for RemConfFS - -- Tim Abbott Sat, 27 Oct 2007 16:54:26 -0400 + -- Evan Broder Wed, 01 Oct 2008 02:21:00 -0400 -sipb-xen-remctl-auto (1.0.5) unstable; urgency=low +sipb-xen-remote-server (0.7) unstable; urgency=low - * move to common /usr/sbin/sipb-xen-* naming - * get rid of dispatch.sh hack - * fix "reboot" to "destroy then create" always, since that's what users will expect - * change remctl calls interface in various ways + * depend on sipb-xen-base - -- Tim Abbott Tue, 16 Oct 2007 00:28:36 -0400 + -- Greg Price Tue, 30 Sep 2008 20:47:05 -0400 -sipb-xen-remctl-auto (1.0.4) unstable; urgency=low +sipb-xen-remote-server (0.6) unstable; urgency=low - * Fix typos. + * sipb_xen_database -> invirt.config - -- Anders Kaseorg Tue, 9 Oct 2007 04:47:37 -0400 + -- Yang Zhang Sun, 3 Aug 2008 00:54:59 -0400 -sipb-xen-remctl-auto (1.0.3) unstable; urgency=low +sipb-xen-remote-server (0.5) unstable; urgency=low - * Prepend d_ to database VMs. + * use invirt config in sipb-xen-remconffs, sipb-xen-remctl-help, + sipb-xen-remote-proxy - -- Eric Price Tue, 9 Oct 2007 02:34:56 -0400 + -- Yang Zhang Sun, 3 Aug 2008 00:32:59 -0400 -sipb-xen-remctl-auto (1.0.2) unstable; urgency=low +sipb-xen-remote-server (0.4) unstable; urgency=low - * Added web interface support code. - * Move things from /etc/remctl/sipb-xen-auto/bin to /usr/sbin - * Quoted lots of variables + * use invirt.config to get hostnames + * refactoring: extracted bcast() function into invirt.remote package - -- Tim Abbott Sun, 7 Oct 2007 17:23:40 -0400 + -- Yang Zhang Sat, 2 Aug 2008 20:34:50 -0400 -sipb-xen-remctl-auto (1.0.1) unstable; urgency=low +sipb-xen-remote-server (0.3) unstable; urgency=low - * Stop scaring me with unquoted variables. + * Fill in all the code to make the package work (mostly weeks ago.) + * sipb-xen-remote-listvms aggregates listvms across hosts (with + y_z, broder, andersk) + * sipb-xen-remctl-help gives help + * sipb-xen-remote-control proxies to where a machine is running + * sipb-xen-remote-create chooses where to boot by load-balancing - -- Anders Kaseorg Sun, 05 Aug 2007 22:11:02 -0400 + -- Greg Price Thu, 24 Jul 2008 21:34:20 -0400 -sipb-xen-remctl-auto (1.0) unstable; urgency=low +sipb-xen-remote-server (0.2) unstable; urgency=low - * Initial release. + * FUSE filesystem for remctl configuration. + * Remove *register calls, since the FUSE keeps things up to date. + * An init script to keep the FUSE fs up. - -- Tim Abbott Sun, 15 Jul 2007 19:37:05 -0400 + -- Greg Price Sat, 10 May 2008 22:04:32 -0400 + +sipb-xen-remote-server (0.1) unstable; urgency=low + + * First draft. + + -- Greg Price Sun, 30 Mar 2008 01:08:50 -0400 diff --git a/debian/compat b/debian/compat index b8626c4..7ed6ff8 100644 --- a/debian/compat +++ b/debian/compat @@ -1 +1 @@ -4 +5 diff --git a/debian/control b/debian/control index 0700a91..50e6222 100644 --- a/debian/control +++ b/debian/control @@ -1,10 +1,24 @@ -Source: invirt-remote-host -Section: net -Priority: extra +Source: invirt-remote +Section: servers +Priority: important Maintainer: Invirt project -Build-Depends: cdbs (>= 0.4.23-1.1), debhelper (>= 4.2.0) +Build-Depends: cdbs (>= 0.4.23-1.1), debhelper (>= 5), + python-all-dev (>=2.3.5-11), python-support (>= 0.5.3), + python-setuptools, python-debian, python-apt Standards-Version: 3.8.0 +Package: invirt-remote-server +Architecture: all +Provides: ${diverted-files}, ${python:Provides} +Conflicts: ${diverted-files} +Depends: ${shlibs:Depends}, ${misc:Depends}, ${python:Depends}, + daemon, debathena-kerberos-config, fuse-utils, openssh-server, + python-routefs, invirt-database, invirt-base, remctl-server, + remctl-client +XB-Python-Version: ${python:Versions} +Description: Invirt remote-control server + This package should be installed to set up the remote-control server. + Package: invirt-remote-host Architecture: all Depends: ${misc:Depends}, remctl-server, invirt-console-host, diff --git a/debian/invirt-remote-host.install b/debian/invirt-remote-host.install index 9da31b3..260c50b 100644 --- a/debian/invirt-remote-host.install +++ b/debian/invirt-remote-host.install @@ -1 +1 @@ -files/* . +host/* . diff --git a/debian/invirt-remote-server.init b/debian/invirt-remote-server.init new file mode 100755 index 0000000..4184917 --- /dev/null +++ b/debian/invirt-remote-server.init @@ -0,0 +1,79 @@ +#! /bin/sh +### BEGIN INIT INFO +# Provides: invirt-remote-server +# Required-Start: $local_fs $remote_fs +# Required-Stop: $local_fs $remote_fs +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: Invirt remctl configuration filesystem +# Description: +### END INIT INFO + +# Author: Invirt project + +PATH=/sbin:/usr/sbin:/bin:/usr/bin +NAME=invirt-remote-server +DESC="the Invirt remctl configuration filesystem" +DAEMON=/usr/sbin/invirt-remconffs +MOUNTPOINT="/etc/remctl/remconffs" +DAEMON_ARGS="-f $MOUNTPOINT" +PIDFILE=/var/run/$NAME.pid +SCRIPTNAME=/etc/init.d/$NAME +GEN_FILES=/etc/remctl/acl/web + +# Exit if the package is not installed +[ -x "$DAEMON" ] || exit 0 + +. /lib/init/gen-files.sh +. /lib/init/std-init.sh + +# +# Function that starts the daemon/service +# +do_start() +{ + # Return + # 0 if daemon has been started + # 1 if daemon was already running + # 2 if daemon could not be started + + # Try to make sure fuse is setup + [ -e /dev/fuse ] || modprobe fuse || return 2 + + if cat /proc/mounts | grep " $MOUNTPOINT " >/dev/null 2>&1; then + return 1 + fi + + gen_files + + daemon -r -O daemon.info -E daemon.err -n $NAME -- $DAEMON $DAEMON_ARGS || return 2 +} + +# +# Function that stops the daemon/service +# +do_stop() +{ + # Return + # 0 if daemon has been stopped + # 1 if daemon was already stopped + # 2 if daemon could not be stopped + # other if a failure occurred + + if ! cat /proc/mounts | grep " $MOUNTPOINT " >/dev/null 2>&1; then + return 1 + fi + + daemon --stop -n $NAME + RETVAL="$?" + [ "$RETVAL" = 2 ] && return 2 + # Many daemons don't delete their pidfiles when they exit. + rm -f $PIDFILE +} + +do_reload() +{ + gen_files +} + +std_init "$1" diff --git a/debian/invirt-remote-server.install b/debian/invirt-remote-server.install new file mode 100644 index 0000000..bde7702 --- /dev/null +++ b/debian/invirt-remote-server.install @@ -0,0 +1,2 @@ +server/* . +debian/tmp/* . diff --git a/debian/pycompat b/debian/pycompat new file mode 100644 index 0000000..0cfbf08 --- /dev/null +++ b/debian/pycompat @@ -0,0 +1 @@ +2 diff --git a/debian/rules b/debian/rules index e6192f6..e90e462 100755 --- a/debian/rules +++ b/debian/rules @@ -1,3 +1,9 @@ #!/usr/bin/make -f +DEB_PYTHON_SYSTEM=pysupport + include /usr/share/cdbs/1/rules/debhelper.mk +include /usr/share/cdbs/1/class/python-distutils.mk + +clean:: + rm -rf invirt.remote.egg-info diff --git a/files/etc/remctl/acl/remote.mako b/host/etc/remctl/acl/remote.mako similarity index 100% rename from files/etc/remctl/acl/remote.mako rename to host/etc/remctl/acl/remote.mako diff --git a/files/etc/remctl/conf.d/invirt-remote-host b/host/etc/remctl/conf.d/invirt-remote-host similarity index 100% rename from files/etc/remctl/conf.d/invirt-remote-host rename to host/etc/remctl/conf.d/invirt-remote-host diff --git a/files/usr/sbin/invirt-listvms b/host/usr/sbin/invirt-listvms similarity index 100% rename from files/usr/sbin/invirt-listvms rename to host/usr/sbin/invirt-listvms diff --git a/files/usr/sbin/invirt-lvm b/host/usr/sbin/invirt-lvm similarity index 100% rename from files/usr/sbin/invirt-lvm rename to host/usr/sbin/invirt-lvm diff --git a/files/usr/sbin/invirt-remote b/host/usr/sbin/invirt-remote similarity index 100% rename from files/usr/sbin/invirt-remote rename to host/usr/sbin/invirt-remote diff --git a/files/usr/sbin/invirt-vmcontrol b/host/usr/sbin/invirt-vmcontrol similarity index 100% rename from files/usr/sbin/invirt-vmcontrol rename to host/usr/sbin/invirt-vmcontrol diff --git a/python/remote/__init__.py b/python/remote/__init__.py new file mode 100644 index 0000000..adaa9b7 --- /dev/null +++ b/python/remote/__init__.py @@ -0,0 +1 @@ +from bcast import bcast diff --git a/python/remote/bcast.py b/python/remote/bcast.py new file mode 100644 index 0000000..6a00b23 --- /dev/null +++ b/python/remote/bcast.py @@ -0,0 +1,22 @@ +from subprocess import PIPE, Popen +from invirt.config import structs as config +import yaml + +def bcast(cmd, hosts = [h.hostname for h in config.hosts]): + """ + Given a command and a list of hostnames or IPs, issue the command to all + the nodes and return a list of (host, output) pairs (the order should be + the same as the order of the hosts). + """ + pipes = [(host, + Popen(['remctl', host, 'remote', 'web', cmd], stdout=PIPE, stderr=PIPE)) + for host in hosts] + outputs = dict((s, p.communicate()) for (s, p) in pipes) + for (s, p) in pipes: + if p.returncode != 0: + if outputs[s][1].startswith('remctl: cannot connect to %s' % s): + del outputs[s] + else: + raise RuntimeError("remctl to host %s returned non-zero exit status %d" + % (s, p.returncode)) + return [(s, yaml.load(o[0], yaml.CSafeLoader)) for (s, o) in outputs.iteritems()] diff --git a/server/etc/remctl/acl/web.mako b/server/etc/remctl/acl/web.mako new file mode 100644 index 0000000..4a320c8 --- /dev/null +++ b/server/etc/remctl/acl/web.mako @@ -0,0 +1,2 @@ +<% from invirt.config import structs as cfg %>\ +daemon/${cfg.web.hostname}@${cfg.authn[0].realm}\ diff --git a/server/etc/remctl/conf.d/invirt-web b/server/etc/remctl/conf.d/invirt-web new file mode 100644 index 0000000..af81d8a --- /dev/null +++ b/server/etc/remctl/conf.d/invirt-web @@ -0,0 +1,3 @@ +web ALL /usr/sbin/invirt-remote-proxy-web /etc/remctl/acl/web +control help /usr/sbin/invirt-remctl-help ANYUSER +help ALL /usr/sbin/invirt-remctl-help ANYUSER diff --git a/server/etc/remctl/conf.d/remconffs b/server/etc/remctl/conf.d/remconffs new file mode 100644 index 0000000..0bda904 --- /dev/null +++ b/server/etc/remctl/conf.d/remconffs @@ -0,0 +1 @@ +include /etc/remctl/remconffs/conf diff --git a/server/usr/sbin/invirt-remconffs b/server/usr/sbin/invirt-remconffs new file mode 100755 index 0000000..6389391 --- /dev/null +++ b/server/usr/sbin/invirt-remconffs @@ -0,0 +1,88 @@ +#!/usr/bin/python + +import routefs +from routes import Mapper + +from syslog import * +from time import time +import sqlalchemy as sa + +from invirt import database +from invirt.config import structs as config + +class RemConfFS(routefs.RouteFS): + """ + RemConfFS creates a filesytem for configuring remctl, like this: + / + |-- acl + | |-- machine1 + | ... + | `-- machinen + `-- conf + + The machine list and the acls are drawn from a database. + """ + + def __init__(self, *args, **kw): + """Initialize the filesystem and set it to allow_other access besides + the user who mounts the filesystem (i.e. root) + """ + super(RemConfFS, self).__init__(*args, **kw) + self.fuse_args.add("allow_other", True) + + openlog('invirt-remconffs ', LOG_PID, LOG_DAEMON) + + syslog(LOG_DEBUG, 'Init complete.') + + def make_map(self): + m = Mapper() + m.connect('', controller='getroot') + m.connect('acl', controller='getmachines') + m.connect('acl/:machine', controller='getacl') + m.connect('conf', controller='getconf') + return m + + def getroot(self, **kw): + return ['acl', 'conf'] + + def getacl(self, machine, **kw): + """Build the ACL file for a machine + """ + s = sa.sql.select([database.machine_access_table.c.user], # Field to select from + sa.sql.and_( # where clause + database.machine_table.c.machine_id==database.machine_access_table.c.machine_id, # join field + database.machine_table.c.name == machine), # filter field + from_obj=[database.machine_access_table, database.machine_table]) # from tables + users = [self.userToPrinc(acl[0]) for acl in + database.session.execute(s)] + return "\n".join(users + + ['include /etc/remctl/acl/web', + '']) + + def getconf(self, **kw): + """Build the master conf file, with all machines + """ + return '\n'.join("control %s /usr/sbin/invirt-remote-proxy-control" + " /etc/remctl/remconffs/acl/%s" + % (machine_name, machine_name) + for machine_name in self.getmachines())+'\n' + + def getmachines(self, **kw): + """Get the list of VMs in the database. Does not cache to prevent race conditions.""" + return list(row[0] for row in database.session.execute(sa.sql.select([database.Machine.c.name]))) + + def userToPrinc(self, user): + """Convert Kerberos v4-style names to v5-style and append a default + realm if none is specified + """ + if '@' in user: + (princ, realm) = user.split('@') + else: + princ = user + realm = config.authn[0].realm + + return princ.replace('.', '/') + '@' + realm + +if __name__ == '__main__': + database.connect() + routefs.main(RemConfFS) diff --git a/server/usr/sbin/invirt-remctl-help b/server/usr/sbin/invirt-remctl-help new file mode 100755 index 0000000..33c67a6 --- /dev/null +++ b/server/usr/sbin/invirt-remctl-help @@ -0,0 +1,42 @@ +#!/usr/bin/python +""" +Help on using the Invirt remctl functions. +""" +import sys +from invirt.config import structs as config + +help = [ + ('list', 'show your VM\'s state (with xm list)'), + ('list-host', 'show on what host, if any, your VM is running'), + ('list-long', 'show your VM\'s state as an sexp (with xm list --long)'), + ('vcpu-list', 'show your VM\'s state (with xm vcpu-list)'), + ('uptime', 'show your VM\'s state (with xm uptime)'), + ('destroy', 'shut down your VM, hard (with xm destroy)'), + ('shutdown', 'shut down your VM, softly if paravm (with xm shutdown)'), + ('create', 'start up your VM (with xm create)'), + ('reboot', 'reboot your VM (with xm reboot)'), + ('install', 'autoinstall your VM (takes a series of key=value pairs; \n\t\tvalid arguments include mirror, dist, arch, imagesize,\n\t\tand noinstall)'), + #also CD images on create/reboot +] +helpdict = dict(help) + + +def print_help(name, text): + print ' %-9s : %s' % (name, text) + +def main(args): + args = [n for n in args if n in helpdict] + print 'remctl %s control ' % config.remote.hostname + if args: + for name in args: + print_help(name, helpdict[name]) + else: + for name, text in help: + print_help(name, text) + + return 0 + +if __name__ == '__main__': + sys.exit(main(sys.argv[1:])) + +# vim:et:sw=4:ts=4 diff --git a/server/usr/sbin/invirt-remote-control b/server/usr/sbin/invirt-remote-control new file mode 100755 index 0000000..b205760 --- /dev/null +++ b/server/usr/sbin/invirt-remote-control @@ -0,0 +1,45 @@ +#!/usr/bin/python +""" +Sends remctl commands about a running VM to the host it's running on. +""" + +from subprocess import PIPE, Popen, call +import sys +import yaml + +def main(argv): + if len(argv) < 3: + print >>sys.stderr, "usage: invirt-remote-control " + return 2 + machine_name = argv[1] + command = argv[2] + + p = Popen(['/usr/sbin/invirt-remote-proxy-web', 'listvms'], stdout=PIPE) + output = p.communicate()[0] + if p.returncode != 0: + raise RuntimeError("Command '%s' returned non-zero exit status %d" + % ('invirt-remote-proxy-web', p.returncode)) + vms = yaml.load(output, yaml.CSafeLoader) + + if machine_name not in vms: + print >>sys.stderr, "machine '%s' is not on" % machine_name + return 1 + host = vms[machine_name]['host'] + + p = Popen(['remctl', host, 'remote', 'control'] + argv[1:], + stdout=PIPE, stderr=PIPE) + (out, err) = p.communicate() + if p.returncode == 1: + print >>sys.stderr, "machine '%s' is not on" % machine_name + return 1 + elif p.returncode == 34: + print >>sys.stderr, "ERROR: invalid command" + return 34 + sys.stderr.write(err) + sys.stdout.write(out) + return p.returncode + +if __name__ == '__main__': + sys.exit(main(sys.argv)) + +# vim:et:sw=4:ts=4 diff --git a/server/usr/sbin/invirt-remote-create b/server/usr/sbin/invirt-remote-create new file mode 100755 index 0000000..0a95d35 --- /dev/null +++ b/server/usr/sbin/invirt-remote-create @@ -0,0 +1,62 @@ +#!/usr/bin/python + +""" +Picks a host to "create" (boot) a VM on, and does so. + +Current load-balancing algorithm: wherever there's more free RAM. + +TODO: use a lock to avoid creating the same VM twice in a race +""" + +from invirt.remote import bcast +from subprocess import PIPE, Popen, call +import sys +import yaml + +def choose_host(): + # Query each of the hosts. + # XXX will the output of 'xm info' always be parseable YAML? + results = bcast('info') + return max((int(o['max_free_memory']), s) for (s, o) in results)[1] + +def main(argv): + if len(argv) < 3: + print >> sys.stderr, "usage: invirt-remote-create []" + return 2 + operation = argv[1] + machine_name = argv[2] + args = argv[3:] + + if operation == 'install': + options = dict(arg.split('=', 1) for arg in args) + valid_keys = set(('mirror', 'dist', 'arch', 'imagesize', 'noinstall')) + if not set(options.keys()).issubset(valid_keys): + print >> sys.stderr, "Invalid argument. Use the help command to see valid arguments to install" + return 1 + if any(' ' in val for val in options.values()): + print >> sys.stderr, "Arguments to the autoinstaller cannot contain spaces" + return 1 + + p = Popen(['/usr/sbin/invirt-remote-proxy-web', 'listvms'], stdout=PIPE) + output = p.communicate()[0] + if p.returncode != 0: + raise RuntimeError("Command '%s' returned non-zero exit status %d" + % ('invirt-remote-proxy-web', p.returncode)) + vms = yaml.load(output, yaml.CSafeLoader) + + if machine_name in vms: + host = vms[machine_name]['host'] + print >> sys.stderr, ("machine '%s' is already running on host %s" + % (machine_name, host)) + return 1 + + host = choose_host() + print 'Creating on host %s...' % host + sys.stdout.flush() + return call(['remctl', host, 'remote', 'control', + machine_name, operation] + args) + +if __name__ == '__main__': + sys.exit(main(sys.argv)) + +# vim:et:sw=4:ts=4 diff --git a/server/usr/sbin/invirt-remote-listhost b/server/usr/sbin/invirt-remote-listhost new file mode 100755 index 0000000..64f6edf --- /dev/null +++ b/server/usr/sbin/invirt-remote-listhost @@ -0,0 +1,33 @@ +#!/usr/bin/python +""" +Say what host a running VM is on. +""" + +from subprocess import PIPE, Popen, call +import sys +import yaml + +def main(argv): + if len(argv) < 2: + print >>sys.stderr, "usage: invirt-remote-listhost " + return 2 + machine_name = argv[1] + + p = Popen(['/usr/sbin/invirt-remote-proxy-web', 'listvms'], stdout=PIPE) + output = p.communicate()[0] + if p.returncode != 0: + raise RuntimeError("Command '%s' returned non-zero exit status %d" + % ('invirt-remote-proxy-web', p.returncode)) + vms = yaml.load(output, yaml.CSafeLoader) + + if machine_name not in vms: + print >>sys.stderr, "machine '%s' is not on" % machine_name + return 2 + + print vms[machine_name]['host'] + return 0 + +if __name__ == '__main__': + sys.exit(main(sys.argv)) + +# vim:et:sw=4:ts=4 diff --git a/server/usr/sbin/invirt-remote-listvms b/server/usr/sbin/invirt-remote-listvms new file mode 100755 index 0000000..8c25d09 --- /dev/null +++ b/server/usr/sbin/invirt-remote-listvms @@ -0,0 +1,28 @@ +#!/usr/bin/python + +""" +Collates the results of listvms from multiple VM servers. Part of the xvm +suite. +""" + +from invirt.remote import bcast +import sys +import yaml + +def main(argv): + # Query each of the hosts. + results = filter(lambda (_, x): x is not None, bcast('listvms')) + + # Merge the results and print. + merged = {} + for server, result in results: + for data in result.itervalues(): + data['host'] = server + merged.update(result) + + print yaml.dump(merged, Dumper=yaml.CSafeDumper, default_flow_style=False) + +if __name__ == '__main__': + sys.exit(main(sys.argv)) + +# vim:et:sw=4:ts=4 diff --git a/server/usr/sbin/invirt-remote-listvmsd b/server/usr/sbin/invirt-remote-listvmsd new file mode 100755 index 0000000..33acef3 --- /dev/null +++ b/server/usr/sbin/invirt-remote-listvmsd @@ -0,0 +1,71 @@ +#!/usr/bin/perl + +# NOTE: In development; not actually used yet. + +#Collates the results of listvms from multiple VM servers. Part of the xvm +#suite. + +use Net::Remctl (); +use JSON; + +our @servers = qw/black-mesa.mit.edu sx-blade-2.mit.edu/; + +our %connections; + +sub openConnections() { + foreach (@servers) { openConnection($_); } +} + +sub openConnection($) { + my ($server) = @_; + my $remctl = Net::Remctl->new; + $remctl->open($server) + or die "Cannot connect to $server: ", $remctl->error, "\n"; + $connections{$server} = $remctl; +} + +sub doListVMs() { + foreach my $remctl (values %connections) { + $remctl->command("remote", "web", "listvms", "--json"); + } + my %vmstate; + foreach my $server (keys %connections) { + my $remctl = $connections{$server}; + my $jsonData = ''; + do { + $output = $remctl->output; + if ($output->type eq 'output') { + if ($output->stream == 1) { + $jsonData .= $output->data; + } elsif ($output->stream == 2) { + print STDERR $output->data; + } + } elsif ($output->type eq 'error') { + warn $output->error, "\n"; + } elsif ($output->type eq 'status') { + if ($output->status != 0) { + warn "Exit status was ".$output->status; + } + } elsif ($output->type eq 'done') { + #next; + } else { + die "Unknown output token from library: ", $output->type, "\n"; + } + } while ($output->type ne 'done'); + my $vmlist = jsonToObj($jsonData); + foreach my $key (keys %$vmlist) { + $vmstate{$key} = $vmlist->{$key}; + $vmstate{$key}{"host"} = $server; + } + } + return %vmstate; +} + +openConnections(); + +use Data::Dumper; +use Benchmark; +print Dumper({doListVMs()}); +timethis(100, sub {doListVMs()}); + +# vim:et:sw=4:ts=4 diff --git a/server/usr/sbin/invirt-remote-proxy b/server/usr/sbin/invirt-remote-proxy new file mode 100755 index 0000000..f05fa65 --- /dev/null +++ b/server/usr/sbin/invirt-remote-proxy @@ -0,0 +1,30 @@ +#!/bin/bash +# invoke as invirt-remote-proxy-$TYPE, with "TYPE" in the remctl sense. + +klist -s || kinit -k + +TYPE="${0##*-}" +case "$TYPE" in + control ) + MACHINE="$1"; SERVICE="$2"; shift; shift ;; + * ) + SERVICE="$1"; shift ;; +esac + +case "$TYPE/$SERVICE" in + web/listvms ) + invirt-remote-listvms "$@" ;; + web/vnccert ) + invirt-remote-vnccert "$@" ;; + control/help ) + invirt-remctl-help ;; + control/create|control/install ) + invirt-remote-create "$SERVICE" "$MACHINE" "$@" ;; + control/listhost|control/list-host ) + invirt-remote-listhost "$MACHINE" "$@" ;; + control/* ) + # Everything but create must go where the VM is already running. + invirt-remote-control "$MACHINE" "$SERVICE" "$@" ;; + * ) + remctl "$(invirt-getconf hosts.0.hostname)" remote "$TYPE" "$SERVICE" "$@" ;; +esac diff --git a/server/usr/sbin/invirt-remote-proxy-control b/server/usr/sbin/invirt-remote-proxy-control new file mode 120000 index 0000000..215b1de --- /dev/null +++ b/server/usr/sbin/invirt-remote-proxy-control @@ -0,0 +1 @@ +invirt-remote-proxy \ No newline at end of file diff --git a/server/usr/sbin/invirt-remote-proxy-web b/server/usr/sbin/invirt-remote-proxy-web new file mode 120000 index 0000000..215b1de --- /dev/null +++ b/server/usr/sbin/invirt-remote-proxy-web @@ -0,0 +1 @@ +invirt-remote-proxy \ No newline at end of file diff --git a/server/usr/sbin/invirt-remote-vnccert b/server/usr/sbin/invirt-remote-vnccert new file mode 100755 index 0000000..019256a --- /dev/null +++ b/server/usr/sbin/invirt-remote-vnccert @@ -0,0 +1,26 @@ +#!/usr/bin/python + +""" +Retrieves the VNC certificate from an Invirt host +""" + +from invirt.config import structs as config +from subprocess import Popen, call, PIPE +import sys + + +def main(argv): + if len(argv) < 2: + print >> sys.stderr, "usage: invirt-remote-vnccert " + return 2 + + host = argv[1] + + if host not in list(i.hostname for i in config.hosts): + print >> sys.stderr, "Invalid hostname specified" + return 1 + + return call(['remctl', host, 'remote', 'web', 'vnccert']) + +if __name__ == '__main__': + sys.exit(main(sys.argv)) diff --git a/setup.py b/setup.py new file mode 100755 index 0000000..f8c1a24 --- /dev/null +++ b/setup.py @@ -0,0 +1,23 @@ +#!/usr/bin/python + +from os import path +from debian_bundle.changelog import Changelog +from debian_bundle.deb822 import Deb822 +from email.utils import parseaddr +from setuptools import setup + +version = Changelog(open(path.join(path.dirname(__file__), 'debian/changelog')).read()).\ + get_version().full_version + +maintainer_full = Deb822(open(path.join(path.dirname(__file__), 'debian/control')))['Maintainer'] +maintainer, maintainer_email = parseaddr(maintainer_full) + +setup( + name='invirt.remote', + version=version, + maintainer=maintainer, + maintainer_email=maintainer_email, + + packages = ['invirt.remote'], + package_dir = {'invirt': 'python'} +)