From: Evan Broder Date: Sat, 28 Feb 2009 00:48:30 +0000 (-0500) Subject: Merge invirt-database-server into invirt-database. X-Git-Tag: 0.2.0^0 X-Git-Url: http://xvm.mit.edu/gitweb/invirt/packages/invirt-database.git/commitdiff_plain/refs/tags/0.2.0?hp=465101869ae825e4556a27b06a3e0c2e84adca3b Merge invirt-database-server into invirt-database. I...think this works, but I'm not making any promises. svn path=/trunk/packages/invirt-database/; revision=2218 --- diff --git a/debian/changelog b/debian/changelog index a13809f..58ee681 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,73 +1,122 @@ -invirt-database-server (0.0.11) unstable; urgency=low +invirt-database (0.2.0) unstable; urgency=low - * bumped version number to force new owner table creation + * Merge invirt-database-server into invirt-database. - -- Peter A. Iannucci Mon, 16 Feb 2009 23:59:53 -0500 + -- Evan Broder Fri, 27 Feb 2009 19:38:55 -0500 -invirt-database-server (0.0.10) unstable; urgency=low +invirt-database (0.1.9) unstable; urgency=low - [ Greg Price ] - * invirt-database-tables: populate types table + * Make get*Quotas idempotent + * Move Owner import so that Owner can access session in + invirt.database.models - [ Evan Broder ] - * Conditionalize populating the types table on the types not being there - already. + -- Quentin Smith Fri, 27 Feb 2009 14:17:53 -0500 - -- Evan Broder Sat, 31 Jan 2009 03:58:16 -0500 +invirt-database (0.1.8) unstable; urgency=low -invirt-database-server (0.0.9) unstable; urgency=low + [Peter Iannucci] + * Added Record superclass for models, handling __repr__ consistently. - * Add Provides and Conflicts lines to debian/control for - config-package-dev + [Greg Price] + invirt.database.record, .owner: + * use self.c rather than self.__dict__ for SQLAlchemy fields + * make Record._ignore, Owner.get* classmethods + * fold FormattableRecord, NullableRecord into Record + * shorten types in lists to __name__ + + invirt-quota: + * refactor code that sets values + * print full help on no arguments - -- Evan Broder Sat, 24 Jan 2009 20:15:08 -0500 + all: + * make EIBTI imports -invirt-database-server (0.0.8) unstable; urgency=low + [ Quentin Smith ] + * Add "reusable" column for nics, to avoid reusing addresses that are + special in some way. - * pg_hba.conf: remove password-auth localhost lines, which interfere - with the case where config.db.host is localhost. + -- Quentin Smith Fri, 27 Feb 2009 11:51:39 -0500 - -- Greg Price Mon, 29 Dec 2008 00:03:36 -0500 +invirt-database (0.1.7) unstable; urgency=low -invirt-database-server (0.0.7) unstable; urgency=low + * Disk quotas are measured in gibibytes. + * Rename invirt-setquotas to invirt-quota. + * Be explicit about units in invirt-quota. - * Don't depend on invirt-mail-config + -- Evan Broder Tue, 17 Feb 2009 04:29:20 -0500 - -- Evan Broder Thu, 06 Nov 2008 22:49:30 -0500 +invirt-database (0.1.6) unstable; urgency=low -invirt-database-server (0.0.6) unstable; urgency=low + * Move invirt-setquotas into this package. + * Don't explicitly initialize the values for each quota when creating a + new Owner object. - * Depend on invirt-mail-config + -- Evan Broder Tue, 17 Feb 2009 03:44:20 -0500 - -- Evan Broder Thu, 06 Nov 2008 21:47:17 -0500 +invirt-database (0.1.5) unstable; urgency=low -invirt-database-server (0.0.5) unstable; urgency=low + * Make the default values on the owners table explicit + (Note: This is a NOOP, since a nullable field with no otherwise + specified default defaults to NULL) - * Use invoke-rc.d instead of calling init scripts directly + -- Evan Broder Tue, 17 Feb 2009 03:36:47 -0500 - -- Evan Broder Fri, 31 Oct 2008 06:26:30 -0400 +invirt-database (0.1.4) unstable; urgency=low -invirt-database-server (0.0.4) unstable; urgency=low + * minor bug fix - * sipb-xen-base -> invirt-base + -- Peter A. Iannucci Tue, 17 Feb 2009 02:59:14 -0500 - -- Evan Broder Tue, 28 Oct 2008 04:23:20 -0400 +invirt-database (0.1.3) unstable; urgency=low -invirt-database-server (0.0.3) unstable; urgency=low + * Added owner table to database with ram, disk, and VM quotas + * Refactored Owner class into separate sourcefile - * Fix a typo on the postfix.conf.invirt transform script - * Postgres will try to intelligently select a port. We don't want it to - do that - force the port to our standard number + -- Peter A. Iannucci Tue, 17 Feb 2009 01:31:53 -0500 - -- Evan Broder Sat, 25 Oct 2008 21:50:02 -0400 +invirt-database (0.1.2) unstable; urgency=low -invirt-database-server (0.0.2) unstable; urgency=low + * Clean up the .egg-info directory from the right place. - * Catch one dependency that wasn't updated + -- Evan Broder Mon, 05 Jan 2009 23:50:47 -0500 - -- Evan Broder Sat, 25 Oct 2008 21:25:58 -0400 +invirt-database (0.1.1) unstable; urgency=low -invirt-database-server (0.0.1) unstable; urgency=low + * Correct a lintian spelling warning. + + -- Evan Broder Mon, 05 Jan 2009 23:40:44 -0500 + +invirt-database (0.1.0) unstable; urgency=low + + * Turn on lazy loading for all tables + + -- Evan Broder Mon, 17 Nov 2008 21:57:35 -0500 + +invirt-database (0.0.6) unstable; urgency=low + + * Let machines.administrator be NULL + + -- Evan Broder Mon, 17 Nov 2008 13:41:48 -0500 + +invirt-database (0.0.5) unstable; urgency=low + + * Add an architecture field to the autoinstaller table + + -- Evan Broder Mon, 17 Nov 2008 12:18:02 -0500 + +invirt-database (0.0.3) unstable; urgency=low + + * Add attributes to the ORM objects to connect mirrors and cdroms + + -- Evan Broder Tue, 28 Oct 2008 03:16:18 -0400 + +invirt-database (0.0.2) unstable; urgency=low + + * Export mirrors_table and Mirrors from invirt.database.models + + -- Evan Broder Tue, 28 Oct 2008 03:04:54 -0400 + +invirt-database (0.0.1) unstable; urgency=low * sipb-xen-database-common -> invirt-database * sipb-xen-database-server -> invirt-database-server (a separate source package) diff --git a/debian/compat b/debian/compat index 7ed6ff8..b8626c4 100644 --- a/debian/compat +++ b/debian/compat @@ -1 +1 @@ -5 +4 diff --git a/debian/control b/debian/control index db33198..a3369d7 100644 --- a/debian/control +++ b/debian/control @@ -1,10 +1,22 @@ -Source: invirt-database-server +Source: invirt-database Section: net Priority: extra Maintainer: Invirt project -Build-Depends: cdbs, debhelper (>= 5), config-package-dev, postgresql-8.3 +Build-Depends: cdbs (>= 0.4.23-1.1), debhelper (>= 4.2.0), + config-package-dev, postgresql-8.3, python-all-dev (>=2.3.5-11), + python-support (>= 0.5.3), python-setuptools, python-debian, + python-apt Standards-Version: 3.8.0 +Package: invirt-database +Architecture: all +Depends: ${misc:Depends}, ${python:Depends}, python-sqlalchemy, + python-psycopg2 +Provides: ${python:Provides} +XB-Python-Version: ${python:Versions} +Description: Installs the Invirt database schema files + This contains the Python modules to access the Invirt database + Package: invirt-database-server Architecture: all Depends: ${misc:Depends}, ${python:Depends}, postgresql-8.3, diff --git a/debian/invirt-database.install b/debian/invirt-database.install new file mode 100644 index 0000000..84cc1ed --- /dev/null +++ b/debian/invirt-database.install @@ -0,0 +1 @@ +debian/tmp/* . diff --git a/debian/pycompat b/debian/pycompat new file mode 100644 index 0000000..0cfbf08 --- /dev/null +++ b/debian/pycompat @@ -0,0 +1 @@ +2 diff --git a/debian/rules b/debian/rules index f707006..d2e9b8f 100755 --- a/debian/rules +++ b/debian/rules @@ -1,5 +1,7 @@ #!/usr/bin/make -f +DEB_PYTHON_SYSTEM=pysupport + DEB_DIVERT_EXTENSION = .invirt DEB_TRANSFORM_FILES_invirt-database-server += \ @@ -9,4 +11,8 @@ DEB_DIVERT_FILES_invirt-database-server += \ /etc/init.d/postgresql-8.3.invirt include /usr/share/cdbs/1/rules/debhelper.mk +include /usr/share/cdbs/1/class/python-distutils.mk include /usr/share/cdbs/1/rules/config-package.mk + +clean:: + rm -rf invirt.database.egg-info diff --git a/python/database/__init__.py b/python/database/__init__.py new file mode 100644 index 0000000..b3dd61c --- /dev/null +++ b/python/database/__init__.py @@ -0,0 +1,9 @@ +from invirt.config import structs as config +from models import * +import sqlalchemy + +def connect(uri = config.db.uri): + """ Connect to a given database URI""" + engine = sqlalchemy.create_engine(uri, pool_timeout=5) + meta.bind = engine + session.configure(bind=engine) diff --git a/python/database/models.py b/python/database/models.py new file mode 100644 index 0000000..420c1ae --- /dev/null +++ b/python/database/models.py @@ -0,0 +1,149 @@ +from sqlalchemy import * +from sqlalchemy import orm +from sqlalchemy.orm import create_session, relation + +from sqlalchemy.ext.sessioncontext import SessionContext +from sqlalchemy.ext.assignmapper import assign_mapper + +from invirt.database import record + +__all__ = ['meta', + 'session', + 'clear_cache', + 'machine_table', + 'machine_access_table', + 'nic_table', + 'disk_table', + 'types_table', + 'cdroms_table', + 'mirrors_table', + 'autoinstalls_table', + 'owners_table', + 'Machine', + 'MachineAccess', + 'NIC', + 'Disk', + 'Type', + 'CDROM', + 'Mirror', + 'Autoinstall', + 'Owner', + 'or_', + ] + +meta = ThreadLocalMetaData() +session = orm.scoped_session(orm.sessionmaker(transactional=False, autoflush=False)) + +machine_table = Table('machines', meta, + Column('machine_id', Integer, primary_key=True, nullable=False), + Column('name', String, nullable=False), + Column('description', String, nullable=False), + Column('memory', Integer, nullable=False), + Column('owner', String, nullable=False), + Column('contact', String, nullable=False), + Column('uuid', String, nullable=False), + Column('administrator', String, nullable=True, default=None), + Column('type_id', String, ForeignKey('types.type_id'), nullable=False), + Column('autorestart', Boolean, nullable=False, default=False), + Column('cpus', Integer, nullable=False, default=1), + Column('adminable', Boolean, nullable=False, default=False)) + +nic_table = Table('nics', meta, + Column('machine_id', Integer, ForeignKey('machines.machine_id'), nullable=True), + Column('mac_addr', String, nullable=False, primary_key=True), + Column('ip', String, nullable=False, unique=True), + Column('hostname', String, nullable=True), + Column('reusable', Boolean, nullable=False, default=True)) + +disk_table = Table('disks', meta, + Column('machine_id', Integer, ForeignKey('machines.machine_id'), nullable=False), + Column('guest_device_name', String, nullable=False), + Column('size', Integer, nullable=False), + PrimaryKeyConstraint('machine_id', 'guest_device_name')) + +types_table = Table('types', meta, + Column('type_id', String, primary_key=True, nullable=False), + Column('description', String, nullable=False), + Column('hvm', Boolean, nullable=False), + Column('apic', Boolean, nullable=False), + Column('acpi', Boolean, nullable=False), + Column('pae', Boolean, nullable=False)) + +mirrors_table = Table('mirrors', meta, + Column('mirror_id', String, primary_key=True, nullable=False), + Column('uri_prefix', String, nullable=False)) + +cdroms_table = Table('cdroms', meta, + Column('cdrom_id', String, primary_key=True, nullable=False), + Column('description', String, nullable=False), + Column('mirror_id', String, ForeignKey('mirrors.mirror_id')), + Column('uri_suffix', String)) + +autoinstalls_table = Table('autoinstalls', meta, + Column('autoinstall_id', String, primary_key=True, nullable=False), + Column('description', String, nullable=False), + Column('type_id', String, ForeignKey('types.type_id'), nullable=False), + Column('distribution', String, nullable=False), + Column('mirror', String, nullable=False), + Column('arch', String, nullable=False)) + +owners_table = Table('owners', meta, + Column('owner_id', String, primary_key=True, nullable=False), + Column('ram_quota_total', Integer, nullable=True, default=None), + Column('ram_quota_single', Integer, nullable=True, default=None), + Column('disk_quota_total', Integer, nullable=True, default=None), + Column('disk_quota_single', Integer, nullable=True, default=None), + Column('vms_quota_total', Integer, nullable=True, default=None), + Column('vms_quota_active', Integer, nullable=True, default=None)) + +machine_access_table = Table('machine_access', meta, + Column('machine_id', Integer, ForeignKey('machines.machine_id', ondelete='CASCADE'), nullable=False, index=True), + Column('user', String, nullable=False, index=True), + PrimaryKeyConstraint('machine_id', 'user')) + +class Machine(record.Record): + _identity_field = 'name' + +class MachineAccess(record.Record): + pass + +class NIC(record.Record): + pass + +class Disk(record.Record): + pass + +class Type(record.Record): + _identity_field = 'type_id' + +class Mirror(record.Record): + _identity_field = 'mirror_id' + +class CDROM(record.Record): + _identity_field = 'cdrom_id' + +class Autoinstall(record.Record): + _identity_field = 'autoinstall_id' + +from invirt.database.owner import Owner + +session.mapper(Machine, machine_table, + properties={'nics': relation(NIC, backref="machine"), + 'disks': relation(Disk, backref="machine"), + 'type': relation(Type), + 'acl': relation(MachineAccess, backref="machine", passive_deletes=True, cascade="all, delete-orphan")}); +session.mapper(MachineAccess, machine_access_table) +session.mapper(NIC, nic_table) +session.mapper(Disk, disk_table) +session.mapper(Type, types_table) +session.mapper(Mirror, mirrors_table) +session.mapper(CDROM, cdroms_table, + properties={'mirror': relation(Mirror, backref="cdroms")}) +session.mapper(Autoinstall, autoinstalls_table) +session.mapper(Owner, owners_table) + +def clear_cache(): + """Clear sqlalchemy's cache + """ + + session.clear() diff --git a/python/database/owner.py b/python/database/owner.py new file mode 100755 index 0000000..ac8ecfd --- /dev/null +++ b/python/database/owner.py @@ -0,0 +1,41 @@ +from invirt.database import record +from invirt.database.models import session + +class Owner(record.Record): + _f = { + 'ram_quota_total': (512, 'MiB'), + 'ram_quota_single': (512, 'MiB'), + 'disk_quota_total': (50, 'GiB'), + 'disk_quota_single': (50, 'GiB'), + 'vms_quota_total': (10, ''), + 'vms_quota_active': (4, '') + } + _default = dict([(_k,_v[0]) for _k,_v in _f.items()]) + def _unitFormatter(unit): + return lambda v:'%s%s'%(v,unit) + _format = dict([(_k,_unitFormatter(_v[1])) for _k,_v in _f.items()]) + _identity_field = 'owner_id' + + @classmethod + def getMemoryQuotas(cls, owner): + owner_info = cls.query().get(owner) + if owner_info == None: + owner_info = cls(owner_id=owner) + session.expunge(owner_info) + return (owner_info.get('ram_quota_total'), owner_info.get('ram_quota_single')) + + @classmethod + def getDiskQuotas(cls, owner): + owner_info = cls.query().get(owner) + if owner_info == None: + owner_info = cls(owner_id=owner) + session.expunge(owner_info) + return (owner_info.get('disk_quota_total'), owner_info.get('disk_quota_single')) + + @classmethod + def getVMQuotas(cls, owner): + owner_info = cls.query().get(owner) + if owner_info == None: + owner_info = cls(owner_id=owner) + session.expunge(owner_info) + return (owner_info.get('vms_quota_total'), owner_info.get('vms_quota_active')) diff --git a/python/database/record.py b/python/database/record.py new file mode 100755 index 0000000..bcf3d45 --- /dev/null +++ b/python/database/record.py @@ -0,0 +1,47 @@ +class Record(object): + _identity_field = None + _default = {} + _format = {} + + def get(self, field): + v = getattr(self, field, None) + if v is None: + return self._default.get(field) + return v + + def _formatField(self, field): + v = self.get(field) + func = self._format.get(field) + if func: + return func(v) + if callable(v): + v = v() + if not hasattr(v, '__iter__'): + return repr(v) + if len(v) == 0: + return '[]' + return '[%d x %s]'%(len(v), type(v[0]).__name__) + + @classmethod + def _ignore(cls): + return [cls._identity_field] + + def _fields(self): + ignore = self._ignore() + keys = sorted(self.c.keys()) + return [(k,self._formatField(k)) for k in keys if k not in ignore] + + def __repr__(self): + classname = self.__class__.__name__ + + if self._identity_field: + identity = self.__dict__.get(self._identity_field) + identity = ' ' + (identity and repr(identity) or 'hash=%X'%hash(self)) + else: + identity = '' + + payload = " ".join(["%s=%s" % (k, v) for k,v in self._fields()]) + if len(payload) > 0: + payload = ": "+payload + + return "<%s%s%s>" % (classname, identity, payload) diff --git a/scripts/invirt-quota b/scripts/invirt-quota new file mode 100755 index 0000000..f0fb16f --- /dev/null +++ b/scripts/invirt-quota @@ -0,0 +1,66 @@ +#!/usr/bin/env python + +""" +invirt-quota allows an administrator to set memory, disk, and VM quotas +for an owner. Invoking with only an owner name returns the current quotas for +that owner. Setting a parameter to -1 restores the default. +""" + +import sys +import optparse + +from invirt import database + +def main(argv): + parser = optparse.OptionParser(usage = '%prog [options]', + description = __doc__.strip()) + parser.add_option('-m', '--ram-total', + type = 'int', + dest = 'ramtotal', + help = 'set total concurrent RAM quota') + parser.add_option('-n', '--ram-single', + type = 'int', + dest = 'ramsingle', + help = 'set single VM RAM quota') + parser.add_option('-d', '--disk-total', + type = 'int', + dest = 'disktotal', + help = 'set total disk quota') + parser.add_option('-e', '--disk-single', + type = 'int', + dest = 'disksingle', + help = 'set single VM disk quota') + parser.add_option('-v', '--vms-total', + type = 'int', + dest = 'vmstotal', + help = 'set total VM quota') + parser.add_option('-w', '--vms-active', + type = 'int', + dest = 'vmsactive', + help = 'set active VM quota') + opts, args = parser.parse_args() + + if len(args) != 1: + parser.print_help(sys.stderr) + return 1 + owner_id = args[0] + database.connect() + database.session.begin() + + owner = database.Owner.query().filter_by(owner_id=owner_id).first() + if owner is None: + owner = database.Owner(owner_id=owner_id) + + for resource, scope in [('ram', 'total'), ('ram', 'single'), + ('disk', 'total'), ('disk', 'single'), + ('vms', 'total'), ('vms', 'active')]: + val = getattr(opts, resource+scope) + if val is not None: + setattr(owner, resource+'_quota_'+scope, val if val >= 0 else None) + + database.session.commit() + print owner + return 0 + +if __name__ == '__main__': + sys.exit(main(sys.argv)) diff --git a/setup.py b/setup.py new file mode 100755 index 0000000..79db2e4 --- /dev/null +++ b/setup.py @@ -0,0 +1,25 @@ +#!/usr/bin/python + +from os import path +from debian_bundle.changelog import Changelog +from debian_bundle.deb822 import Deb822 +from email.utils import parseaddr +from glob import glob +from setuptools import setup + +version = Changelog(open(path.join(path.dirname(__file__), 'debian/changelog')).read()).\ + get_version().full_version + +maintainer_full = Deb822(open(path.join(path.dirname(__file__), 'debian/control')))['Maintainer'] +maintainer, maintainer_email = parseaddr(maintainer_full) + +setup( + name='invirt.database', + version=version, + maintainer=maintainer, + maintainer_email=maintainer_email, + + packages = ['invirt.database'], + package_dir = {'invirt': 'python'}, + scripts=glob('scripts/*') +)