forum/models/base.py

   1 import datetime
   2 import re
   3 try:
   4     from hashlib import md5
   5 except:
   6     from md5 import new as md5
   7 from urllib import quote_plus, urlencode
   8 from django.db import models, IntegrityError, connection, transaction
   9 from django.utils.http import urlquote  as django_urlquote
  10 from django.utils.html import strip_tags
  11 from django.conf import settings as django_settings
  12 from django.core.urlresolvers import reverse
  13 from django.contrib.contenttypes import generic
  14 from django.contrib.contenttypes.models import ContentType
  15 from django.core.cache import cache
  16 from django.template.defaultfilters import slugify
  17 from django.db.models.signals import post_delete, post_save, pre_save, pre_delete
  18 from django.utils.translation import ugettext as _
  19 from django.utils.safestring import mark_safe
  20 from django.utils.encoding import force_unicode
  21 from django.contrib.sitemaps import ping_google
  22 import django.dispatch
  23 from forum import settings
  24 import logging
  25
  26
  27 if not hasattr(cache, 'get_many'):
  28     #put django 1.2 code here
  29     pass
  30
  31 class LazyQueryList(object):
  32     def __init__(self, model, items):
  33         self.items = items
  34         self.model = model
  35
  36     def __getitem__(self, k):
  37         return self.model.objects.get(id=self.items[k][0])
  38
  39     def __iter__(self):
  40         for id in self.items:
  41             yield self.model.objects.get(id=id[0])
  42
  43     def __len__(self):
  44         return len(self.items)
  45
  46 class ToFetch(unicode):
  47     pass
  48
  49 class CachedQuerySet(models.query.QuerySet):
  50
  51     def lazy(self):
  52         if not len(self.query.aggregates):
  53             values_list = ['id']
  54
  55             if len(self.query.extra):
  56                 extra_keys = self.query.extra.keys()
  57                 values_list += extra_keys
  58
  59             return LazyQueryList(self.model, list(self.values_list(*values_list)))
  60         else:
  61             return self
  62
  63     def obj_from_datadict(self, datadict):
  64         obj = self.model()
  65         obj.__dict__.update(datadict)
  66
  67         if hasattr(obj, '_state'):
  68             obj._state.db = 'default'
  69
  70         return obj
  71
  72     def _base_clone(self):
  73         return self._clone(klass=models.query.QuerySet)
  74
  75     def get(self, *args, **kwargs):
  76         key = self.model.infer_cache_key(kwargs)
  77
  78         if key is not None:
  79             obj = cache.get(key)
  80
  81             if obj is None:
  82                 obj = self._base_clone().get(*args, **kwargs)
  83                 obj.cache()
  84             else:
  85                 obj = self.obj_from_datadict(obj)
  86
  87             obj.reset_original_state()
  88
  89             return obj
  90
  91         return self._base_clone().get(*args, **kwargs)
  92
  93     def _fetch_from_query_cache(self, key):
  94         invalidation_key = self.model._get_cache_query_invalidation_key()
  95         cached_result = cache.get_many([invalidation_key, key])
  96
  97         if not invalidation_key in cached_result:
  98             self.model._set_query_cache_invalidation_timestamp()
  99             return None
 100
 101         if (key in cached_result) and(cached_result[invalidation_key] < cached_result[key][0]):
 102             return cached_result[key][1]
 103
 104         return None
 105
 106     def count(self):
 107         cache_key = self.model._generate_cache_key("CNT:%s" % self._get_query_hash())
 108         result = self._fetch_from_query_cache(cache_key)
 109
 110         if result is not None:
 111             return result
 112
 113         result = super(CachedQuerySet, self).count()
 114         cache.set(cache_key, (datetime.datetime.now(), result), 60 * 60)
 115         return result
 116
 117     def iterator(self):
 118         cache_key = self.model._generate_cache_key("QUERY:%s" % self._get_query_hash())
 119         on_cache_query_attr = self.model.value_to_list_on_cache_query()
 120
 121         to_return = None
 122         to_cache = {}
 123
 124         with_aggregates = len(self.query.aggregates) > 0
 125         key_list = self._fetch_from_query_cache(cache_key)
 126
 127         if key_list is None:
 128             if not with_aggregates:
 129                 values_list = [on_cache_query_attr]
 130
 131                 if len(self.query.extra):
 132                     values_list += self.query.extra.keys()
 133
 134                 key_list = [v[0] for v in self.values_list(*values_list)]
 135                 to_cache[cache_key] = (datetime.datetime.now(), key_list)
 136             else:
 137                 to_return = list(super(CachedQuerySet, self).iterator())
 138                 to_cache[cache_key] = (datetime.datetime.now(), [
 139                     (row.__dict__[on_cache_query_attr], dict([(k, row.__dict__[k]) for k in self.query.aggregates.keys()]))
 140                     for row in to_return])
 141         elif with_aggregates:
 142             tmp = key_list
 143             key_list = [k[0] for k in tmp]
 144             with_aggregates = [k[1] for k in tmp]
 145             del tmp
 146
 147         if (not to_return) and key_list:
 148             row_keys = [self.model.infer_cache_key({on_cache_query_attr: attr}) for attr in key_list]
 149             cached = cache.get_many(row_keys)
 150
 151             to_return = [
 152                 (ck in cached) and self.obj_from_datadict(cached[ck]) or ToFetch(force_unicode(key_list[i])) for i, ck in enumerate(row_keys)
 153             ]
 154
 155             if len(cached) != len(row_keys):
 156                 to_fetch = [unicode(tr) for tr in to_return if isinstance(tr, ToFetch)]
 157
 158                 fetched = dict([(force_unicode(r.__dict__[on_cache_query_attr]), r) for r in
 159                               models.query.QuerySet(self.model).filter(**{"%s__in" % on_cache_query_attr: to_fetch})])
 160
 161                 to_return = [(isinstance(tr, ToFetch) and fetched[unicode(tr)] or tr) for tr in to_return]
 162                 to_cache.update(dict([(self.model.infer_cache_key({on_cache_query_attr: attr}), r._as_dict()) for attr, r in fetched.items()]))
 163
 164             if with_aggregates:
 165                 for i, r in enumerate(to_return):
 166                     r.__dict__.update(with_aggregates[i])
 167
 168
 169         if len(to_cache):
 170             cache.set_many(to_cache, 60 * 60)
 171
 172         if to_return:
 173             for row in to_return:
 174                 if hasattr(row, 'leaf'):
 175                     row = row.leaf
 176
 177                 row.reset_original_state()
 178                 yield row
 179
 180     def _get_query_hash(self):
 181         try:
 182             return md5(unicode(self.query).encode("utf-8")).hexdigest()
 183         except:
 184             return md5(self.query).hexdigest()
 185
 186
 187
 188 class CachedManager(models.Manager):
 189     use_for_related_fields = True
 190
 191     def get_query_set(self):
 192         return CachedQuerySet(self.model)
 193
 194     def get_or_create(self, *args, **kwargs):
 195         try:
 196             return self.get(*args, **kwargs)
 197         except:
 198             return super(CachedManager, self).get_or_create(*args, **kwargs)
 199
 200
 201 class DenormalizedField(object):
 202     def __init__(self, manager, *args, **kwargs):
 203         self.manager = manager
 204         self.filter = (args, kwargs)
 205
 206     def setup_class(self, cls, name):
 207         dict_name = '_%s_dencache_' % name
 208
 209         def getter(inst):
 210             val = inst.__dict__.get(dict_name, None)
 211
 212             if val is None:
 213                 val = getattr(inst, self.manager).filter(*self.filter[0], **self.filter[1]).count()
 214                 inst.__dict__[dict_name] = val
 215                 inst.cache()
 216
 217             return val
 218
 219         def reset_cache(inst):
 220             inst.__dict__.pop(dict_name, None)
 221             inst.uncache()
 222
 223         cls.add_to_class(name, property(getter))
 224         cls.add_to_class("reset_%s_cache" % name, reset_cache)
 225
 226
 227 class BaseMetaClass(models.Model.__metaclass__):
 228     to_denormalize = []
 229
 230     def __new__(cls, *args, **kwargs):
 231         new_cls = super(BaseMetaClass, cls).__new__(cls, *args, **kwargs)
 232
 233         BaseMetaClass.to_denormalize.extend(
 234             [(new_cls, name, field) for name, field in new_cls.__dict__.items() if isinstance(field, DenormalizedField)]
 235         )
 236
 237         return new_cls
 238
 239     @classmethod
 240     def setup_denormalizes(cls):
 241         for new_cls, name, field in BaseMetaClass.to_denormalize:
 242             field.setup_class(new_cls, name)
 243
 244
 245 class BaseModel(models.Model):
 246     __metaclass__ = BaseMetaClass
 247
 248     objects = CachedManager()
 249
 250     class Meta:
 251         abstract = True
 252         app_label = 'forum'
 253
 254     def __init__(self, *args, **kwargs):
 255         super(BaseModel, self).__init__(*args, **kwargs)
 256         self.reset_original_state(kwargs.keys())
 257
 258     def reset_original_state(self, reset_fields=None):
 259         self._original_state = self._as_dict()
 260
 261         if reset_fields:
 262             self._original_state.update(dict([(f, None) for f in reset_fields]))
 263
 264     def get_dirty_fields(self):
 265         return [f.name for f in self._meta.fields if self._original_state[f.attname] != self.__dict__[f.attname]]
 266
 267     def _as_dict(self):
 268         return dict([(name, getattr(self, name)) for name in
 269                      ([f.attname for f in self._meta.fields] + [k for k in self.__dict__.keys() if k.endswith('_dencache_')])
 270         ])
 271
 272     def _get_update_kwargs(self):
 273         return dict([
 274             (f.name, getattr(self, f.name)) for f in self._meta.fields if self._original_state[f.attname] != self.__dict__[f.attname]
 275         ])
 276
 277     def save(self, full_save=False, *args, **kwargs):
 278         put_back = [k for k, v in self.__dict__.items() if isinstance(v, models.expressions.ExpressionNode)]
 279
 280         if hasattr(self, '_state'):
 281             self._state.db = 'default'
 282
 283         if self.id and not full_save:
 284             self.__class__.objects.filter(id=self.id).update(**self._get_update_kwargs())
 285         else:
 286             super(BaseModel, self).save()
 287
 288         if put_back:
 289             try:
 290                 self.__dict__.update(
 291                     self.__class__.objects.filter(id=self.id).values(*put_back)[0]
 292                 )
 293             except:
 294                 logging.error("Unable to read %s from %s" % (", ".join(put_back), self.__class__.__name__))
 295                 self.uncache()
 296
 297         self.reset_original_state()
 298         self._set_query_cache_invalidation_timestamp()
 299         self.cache()
 300
 301     @classmethod
 302     def _get_cache_query_invalidation_key(cls):
 303         return cls._generate_cache_key("INV_TS")
 304
 305     @classmethod
 306     def _set_query_cache_invalidation_timestamp(cls):
 307         cache.set(cls._get_cache_query_invalidation_key(), datetime.datetime.now(), 60 * 60 * 24)
 308
 309         for base in filter(lambda c: issubclass(c, BaseModel) and (not c is BaseModel), cls.__bases__):
 310             base._set_query_cache_invalidation_timestamp()
 311
 312     @classmethod
 313     def _generate_cache_key(cls, key, group=None):
 314         if group is None:
 315             group = cls.__name__
 316
 317         return '%s:%s:%s' % (settings.APP_URL, group, key)
 318
 319     def cache_key(self):
 320         return self._generate_cache_key(self.id)
 321
 322     @classmethod
 323     def value_to_list_on_cache_query(cls):
 324         return 'id'
 325
 326     @classmethod
 327     def infer_cache_key(cls, querydict):
 328         try:
 329             pk = [v for (k,v) in querydict.items() if k in ('pk', 'pk__exact', 'id', 'id__exact'
 330                             ) or k.endswith('_ptr__pk') or k.endswith('_ptr__id')][0]
 331
 332             cache_key = cls._generate_cache_key(pk)
 333
 334             if len(cache_key) > django_settings.CACHE_MAX_KEY_LENGTH:
 335                 cache_key = cache_key[:django_settings.CACHE_MAX_KEY_LENGTH]
 336
 337             return cache_key
 338         except:
 339             return None
 340
 341     def cache(self):
 342         cache.set(self.cache_key(), self._as_dict(), 60 * 60)
 343
 344     def uncache(self):
 345         cache.delete(self.cache_key())
 346
 347     def delete(self):
 348         self.uncache()
 349         self._set_query_cache_invalidation_timestamp()
 350         super(BaseModel, self).delete()
 351
 352
 353 from user import User
 354 from node import Node, NodeRevision, NodeManager
 355 from action import Action
 356
 357
 358
 359
 360