frontend/afe/model_logic.py

"""
Extensions to Django's model logic.
"""

from django.db import models as dbmodels, backend, connection
from django.utils import datastructures
from frontend.afe import readonly_connection

class ValidationError(Exception):
    """\
    Data validation error in adding or updating an object.  The associated
    value is a dictionary mapping field names to error strings.
    """


def _wrap_with_readonly(method):
        def wrapper_method(*args, **kwargs):
            readonly_connection.connection.set_django_connection()
            try:
                return method(*args, **kwargs)
            finally:
                readonly_connection.connection.unset_django_connection()
        wrapper_method.__name__ = method.__name__
        return wrapper_method


def _wrap_generator_with_readonly(generator):
    """
    We have to wrap generators specially.  Assume it performs
    the query on the first call to next().
    """
    def wrapper_generator(*args, **kwargs):
        generator_obj = generator(*args, **kwargs)
        readonly_connection.connection.set_django_connection()
        try:
            first_value = generator_obj.next()
        finally:
            readonly_connection.connection.unset_django_connection()
        yield first_value

        while True:
            yield generator_obj.next()

    wrapper_generator.__name__ = generator.__name__
    return wrapper_generator


def _make_queryset_readonly(queryset):
    """
    Wrap all methods that do database queries with a readonly connection.
    """
    db_query_methods = ['count', 'get', 'get_or_create', 'latest', 'in_bulk',
                        'delete']
    for method_name in db_query_methods:
        method = getattr(queryset, method_name)
        wrapped_method = _wrap_with_readonly(method)
        setattr(queryset, method_name, wrapped_method)

    queryset.iterator = _wrap_generator_with_readonly(queryset.iterator)


class ReadonlyQuerySet(dbmodels.query.QuerySet):
    """
    QuerySet object that performs all database queries with the read-only
    connection.
    """
    def __init__(self, model=None):
        super(ReadonlyQuerySet, self).__init__(model)
        _make_queryset_readonly(self)


    def values(self, *fields):
        return self._clone(klass=ReadonlyValuesQuerySet, _fields=fields)


class ReadonlyValuesQuerySet(dbmodels.query.ValuesQuerySet):
    def __init__(self, model=None):
        super(ReadonlyValuesQuerySet, self).__init__(model)
        _make_queryset_readonly(self)


class ExtendedManager(dbmodels.Manager):
    """\
    Extended manager supporting subquery filtering.
    """

    class _CustomJoinQ(dbmodels.Q):
        """
        Django "Q" object supporting a custom suffix for join aliases.See
        filter_custom_join() for why this can be useful.
        """

        def __init__(self, join_suffix, **kwargs):
            super(ExtendedManager._CustomJoinQ, self).__init__(**kwargs)
            self._join_suffix = join_suffix


        @staticmethod
        def _substitute_aliases(renamed_aliases, condition):
            for old_alias, new_alias in renamed_aliases:
                    condition = condition.replace(backend.quote_name(old_alias),
                                                  backend.quote_name(new_alias))
            return condition


        @staticmethod
        def _unquote_name(name):
            'This may be MySQL specific'
            if backend.quote_name(name) == name:
                return name[1:-1]
            return name


        def get_sql(self, opts):
            joins, where, params = (
                super(ExtendedManager._CustomJoinQ, self).get_sql(opts))

            new_joins = datastructures.SortedDict()

            # rename all join aliases and correct references in later joins
            renamed_tables = []
            # using iteritems seems to mess up the ordering here
            for alias, (table, join_type, condition) in joins.items():
                alias = self._unquote_name(alias)
                new_alias = alias + self._join_suffix
                renamed_tables.append((alias, new_alias))
                condition = self._substitute_aliases(renamed_tables, condition)
                new_alias = backend.quote_name(new_alias)
                new_joins[new_alias] = (table, join_type, condition)

            # correct references in where
            new_where = []
            for clause in where:
                new_where.append(
                    self._substitute_aliases(renamed_tables, clause))

            return new_joins, new_where, params


    def filter_custom_join(self, join_suffix, **kwargs):
        """
        Just like Django filter(), but allows the user to specify a custom
        suffix for the join aliases involves in the filter.  This makes it
        possible to join against a table multiple times (as long as a different
        suffix is used each time), which is necessary for certain queries.
        """
        filter_object = self._CustomJoinQ(join_suffix, **kwargs)
        return self.complex_filter(filter_object)


    @staticmethod
    def _get_quoted_field(table, field):
        return (backend.quote_name(table) + '.' + backend.quote_name(field))


    def _get_key_on_this_table(self, key_field=None):
        if key_field is None:
            # default to primary key
            key_field = self.model._meta.pk.column
        return self._get_quoted_field(self.model._meta.db_table, key_field)



class ValidObjectsManager(ExtendedManager):
    """
    Manager returning only objects with invalid=False.
    """
    def get_query_set(self):
        queryset = super(ValidObjectsManager, self).get_query_set()
        return queryset.filter(invalid=False)


class ModelExtensions(object):
    """\
    Mixin with convenience functions for models, built on top of the
    default Django model functions.
    """
    # TODO: at least some of these functions really belong in a custom
    # Manager class

    field_dict = None
    # subclasses should override if they want to support smart_get() by name
    name_field = None


    @classmethod
    def get_field_dict(cls):
        if cls.field_dict is None:
            cls.field_dict = {}
            for field in cls._meta.fields:
                cls.field_dict[field.name] = field
        return cls.field_dict


    @classmethod
    def clean_foreign_keys(cls, data):
        """\
        -Convert foreign key fields in data from <field>_id to just
        <field>.
        -replace foreign key objects with their IDs
        This method modifies data in-place.
        """
        for field in cls._meta.fields:
            if not field.rel:
                continue
            if (field.attname != field.name and
                field.attname in data):
                data[field.name] = data[field.attname]
                del data[field.attname]
            value = data[field.name]
            if isinstance(value, dbmodels.Model):
                data[field.name] = value.id


    # TODO(showard) - is there a way to not have to do this?
    @classmethod
    def provide_default_values(cls, data):
        """\
        Provide default values for fields with default values which have
        nothing passed in.

        For CharField and TextField fields with "blank=True", if nothing
        is passed, we fill in an empty string value, even if there's no
        default set.
        """
        new_data = dict(data)
        field_dict = cls.get_field_dict()
        for name, obj in field_dict.iteritems():
            if data.get(name) is not None:
                continue
            if obj.default is not dbmodels.fields.NOT_PROVIDED:
                new_data[name] = obj.default
            elif (isinstance(obj, dbmodels.CharField) or
                  isinstance(obj, dbmodels.TextField)):
                new_data[name] = ''
        return new_data


    @classmethod
    def convert_human_readable_values(cls, data, to_human_readable=False):
        """\
        Performs conversions on user-supplied field data, to make it
        easier for users to pass human-readable data.

        For all fields that have choice sets, convert their values
        from human-readable strings to enum values, if necessary.  This
        allows users to pass strings instead of the corresponding
        integer values.

        For all foreign key fields, call smart_get with the supplied
        data.  This allows the user to pass either an ID value or
        the name of the object as a string.

        If to_human_readable=True, perform the inverse - i.e. convert
        numeric values to human readable values.

        This method modifies data in-place.
        """
        field_dict = cls.get_field_dict()
        for field_name in data:
            if data[field_name] is None:
                continue
            field_obj = field_dict[field_name]
            # convert enum values
            if field_obj.choices:
                for choice_data in field_obj.choices:
                    # choice_data is (value, name)
                    if to_human_readable:
                        from_val, to_val = choice_data
                    else:
                        to_val, from_val = choice_data
                    if from_val == data[field_name]:
                        data[field_name] = to_val
                        break
            # convert foreign key values
            elif field_obj.rel:
                dest_obj = field_obj.rel.to.smart_get(
                    data[field_name])
                if (to_human_readable and
                    dest_obj.name_field is not None):
                    data[field_name] = (
                        getattr(dest_obj,
                                dest_obj.name_field))
                else:
                    data[field_name] = dest_obj.id


    @classmethod
    def validate_field_names(cls, data):
        'Checks for extraneous fields in data.'
        errors = {}
        field_dict = cls.get_field_dict()
        for field_name in data:
            if field_name not in field_dict:
                errors[field_name] = 'No field of this name'
        return errors


    @classmethod
    def prepare_data_args(cls, data, kwargs):
        'Common preparation for add_object and update_object'
        data = dict(data) # don't modify the default keyword arg
        data.update(kwargs)
        # must check for extraneous field names here, while we have the
        # data in a dict
        errors = cls.validate_field_names(data)
        if errors:
            raise ValidationError(errors)
        cls.convert_human_readable_values(data)
        return data


    def validate_unique(self):
        """\
        Validate that unique fields are unique.  Django manipulators do
        this too, but they're a huge pain to use manually.  Trust me.
        """
        errors = {}
        cls = type(self)
        field_dict = self.get_field_dict()
        manager = cls.get_valid_manager()
        for field_name, field_obj in field_dict.iteritems():
            if not field_obj.unique:
                continue

            value = getattr(self, field_name)
            existing_objs = manager.filter(**{field_name : value})
            num_existing = existing_objs.count()

            if num_existing == 0:
                continue
            if num_existing == 1 and existing_objs[0].id == self.id:
                continue
            errors[field_name] = (
                'This value must be unique (%s)' % (value))
        return errors


    def do_validate(self):
        errors = self.validate()
        unique_errors = self.validate_unique()
        for field_name, error in unique_errors.iteritems():
            errors.setdefault(field_name, error)
        if errors:
            raise ValidationError(errors)


    # actually (externally) useful methods follow

    @classmethod
    def add_object(cls, data={}, **kwargs):
        """\
        Returns a new object created with the given data (a dictionary
        mapping field names to values). Merges any extra keyword args
        into data.
        """
        data = cls.prepare_data_args(data, kwargs)
        data = cls.provide_default_values(data)
        obj = cls(**data)
        obj.do_validate()
        obj.save()
        return obj


    def update_object(self, data={}, **kwargs):
        """\
        Updates the object with the given data (a dictionary mapping
        field names to values).  Merges any extra keyword args into
        data.
        """
        data = self.prepare_data_args(data, kwargs)
        for field_name, value in data.iteritems():
            if value is not None:
                setattr(self, field_name, value)
        self.do_validate()
        self.save()


    @classmethod
    def query_objects(cls, filter_data, valid_only=True):
        """\
        Returns a QuerySet object for querying the given model_class
        with the given filter_data.  Optional special arguments in
        filter_data include:
        -query_start: index of first return to return
        -query_limit: maximum number of results to return
        -sort_by: list of fields to sort on.  prefixing a '-' onto a
         field name changes the sort to descending order.
        -extra_args: keyword args to pass to query.extra() (see Django
         DB layer documentation)
         -extra_where: extra WHERE clause to append
        """
        query_start = filter_data.pop('query_start', None)
        query_limit = filter_data.pop('query_limit', None)
        if query_start and not query_limit:
            raise ValueError('Cannot pass query_start without '
                             'query_limit')
        sort_by = filter_data.pop('sort_by', [])
        extra_args = filter_data.pop('extra_args', {})
        extra_where = filter_data.pop('extra_where', None)
        if extra_where:
            extra_args.setdefault('where', []).append(extra_where)

        # filters
        query_dict = {}
        for field, value in filter_data.iteritems():
            query_dict[field] = value
        if valid_only:
            manager = cls.get_valid_manager()
        else:
            manager = cls.objects
        query = manager.filter(**query_dict).distinct()

        # other arguments
        if extra_args:
            query = query.extra(**extra_args)
            query = query._clone(klass=ReadonlyQuerySet)

        # sorting + paging
        assert isinstance(sort_by, list) or isinstance(sort_by, tuple)
        query = query.order_by(*sort_by)
        if query_start is not None and query_limit is not None:
            query_limit += query_start
        return query[query_start:query_limit]


    @classmethod
    def query_count(cls, filter_data):
        """\
        Like query_objects, but retreive only the count of results.
        """
        filter_data.pop('query_start', None)
        filter_data.pop('query_limit', None)
        return cls.query_objects(filter_data).count()


    @classmethod
    def clean_object_dicts(cls, field_dicts):
        """\
        Take a list of dicts corresponding to object (as returned by
        query.values()) and clean the data to be more suitable for
        returning to the user.
        """
        for i in range(len(field_dicts)):
            cls.clean_foreign_keys(field_dicts[i])
            cls.convert_human_readable_values(
                field_dicts[i], to_human_readable=True)


    @classmethod
    def list_objects(cls, filter_data):
        """\
        Like query_objects, but return a list of dictionaries.
        """
        query = cls.query_objects(filter_data)
        field_dicts = list(query.values())
        cls.clean_object_dicts(field_dicts)
        return field_dicts


    @classmethod
    def smart_get(cls, *args, **kwargs):
        """\
        smart_get(integer) -> get object by ID
        smart_get(string) -> get object by name_field
        smart_get(keyword args) -> normal ModelClass.objects.get()
        """
        assert bool(args) ^ bool(kwargs)
        if args:
            assert len(args) == 1
            arg = args[0]
            if isinstance(arg, int) or isinstance(arg, long):
                return cls.objects.get(id=arg)
            if isinstance(arg, str) or isinstance(arg, unicode):
                return cls.objects.get(
                    **{cls.name_field : arg})
            raise ValueError(
                'Invalid positional argument: %s (%s)' % (
                str(arg), type(arg)))
        return cls.objects.get(**kwargs)


    def get_object_dict(self):
        """\
        Return a dictionary mapping fields to this object's values.
        """
        object_dict = dict((field_name, getattr(self, field_name))
                           for field_name
                           in self.get_field_dict().iterkeys())
        self.clean_object_dicts([object_dict])
        return object_dict


    @classmethod
    def get_valid_manager(cls):
        return cls.objects


class ModelWithInvalid(ModelExtensions):
    """
    Overrides model methods save() and delete() to support invalidation in
    place of actual deletion.  Subclasses must have a boolean "invalid"
    field.
    """

    def save(self):
        # see if this object was previously added and invalidated
        my_name = getattr(self, self.name_field)
        filters = {self.name_field : my_name, 'invalid' : True}
        try:
            old_object = self.__class__.objects.get(**filters)
        except self.DoesNotExist:
            # no existing object
            super(ModelWithInvalid, self).save()
            return

        self.id = old_object.id
        super(ModelWithInvalid, self).save()


    def clean_object(self):
        """
        This method is called when an object is marked invalid.
        Subclasses should override this to clean up relationships that
        should no longer exist if the object were deleted."""
        pass


    def delete(self):
        assert not self.invalid
        self.invalid = True
        self.save()
        self.clean_object()


    @classmethod
    def get_valid_manager(cls):
        return cls.valid_objects


    class Manipulator(object):
        """
        Force default manipulators to look only at valid objects -
        otherwise they will match against invalid objects when checking
        uniqueness.
        """
        @classmethod
        def _prepare(cls, model):
            super(ModelWithInvalid.Manipulator, cls)._prepare(model)
            cls.manager = model.valid_objects