forum/models/base.py

   1 import datetime
   2 import re
   3 try:
   4     from hashlib import md5
   5 except:
   6     from md5 import new as md5
   7 from urllib import quote_plus, urlencode
   8 from django.db import models, IntegrityError, connection, transaction
   9 from django.utils.http import urlquote  as django_urlquote
  10 from django.utils.html import strip_tags
  11 from django.core.urlresolvers import reverse
  12 from django.contrib.contenttypes import generic
  13 from django.contrib.contenttypes.models import ContentType
  14 from django.core.cache import cache
  15 from django.template.defaultfilters import slugify
  16 from django.db.models.signals import post_delete, post_save, pre_save, pre_delete
  17 from django.utils.translation import ugettext as _
  18 from django.utils.safestring import mark_safe
  19 from django.utils.encoding import force_unicode
  20 from django.contrib.sitemaps import ping_google
  21 import django.dispatch
  22 from forum import settings
  23 import logging
  24
  25
  26 if not hasattr(cache, 'get_many'):
  27     #put django 1.2 code here
  28     pass
  29
  30 class LazyQueryList(object):
  31     def __init__(self, model, items):
  32         self.items = items
  33         self.model = model
  34
  35     def __getitem__(self, k):
  36         return self.model.objects.get(id=self.items[k][0])
  37
  38     def __iter__(self):
  39         for id in self.items:
  40             yield self.model.objects.get(id=id[0])
  41
  42     def __len__(self):
  43         return len(self.items)
  44
  45 class ToFetch(unicode):
  46     pass
  47
  48 class CachedQuerySet(models.query.QuerySet):
  49
  50     def lazy(self):
  51         if not len(self.query.aggregates):
  52             values_list = ['id']
  53
  54             if len(self.query.extra):
  55                 extra_keys = self.query.extra.keys()
  56                 values_list += extra_keys
  57
  58             return LazyQueryList(self.model, list(self.values_list(*values_list)))
  59         else:
  60             return self
  61
  62     def obj_from_datadict(self, datadict):
  63         obj = self.model()
  64         obj.__dict__.update(datadict)
  65
  66         if hasattr(obj, '_state'):
  67             obj._state.db = 'default'
  68
  69         return obj
  70
  71     def _base_clone(self):
  72         return self._clone(klass=models.query.QuerySet)
  73
  74     def get(self, *args, **kwargs):
  75         key = self.model.infer_cache_key(kwargs)
  76
  77         if key is not None:
  78             obj = cache.get(key)
  79
  80             if obj is None:
  81                 obj = self._base_clone().get(*args, **kwargs)
  82                 obj.cache()
  83             else:
  84                 obj = self.obj_from_datadict(obj)
  85                 obj.reset_original_state()
  86
  87             return obj
  88
  89         return self._base_clone().get(*args, **kwargs)
  90
  91     def _fetch_from_query_cache(self, key):
  92         invalidation_key = self.model._get_cache_query_invalidation_key()
  93         cached_result = cache.get_many([invalidation_key, key])
  94
  95         if not invalidation_key in cached_result:
  96             self.model._set_query_cache_invalidation_timestamp()
  97             return None
  98
  99         if (key in cached_result) and(cached_result[invalidation_key] < cached_result[key][0]):
 100             return cached_result[key][1]
 101
 102         return None
 103
 104     def count(self):
 105         cache_key = self.model._generate_cache_key("CNT:%s" % self._get_query_hash())
 106         result = self._fetch_from_query_cache(cache_key)
 107
 108         if result is not None:
 109             return result
 110
 111         result = super(CachedQuerySet, self).count()
 112         cache.set(cache_key, (datetime.datetime.now(), result), 60 * 60)
 113         return result
 114
 115     def iterator(self):
 116         cache_key = self.model._generate_cache_key("QUERY:%s" % self._get_query_hash())
 117         on_cache_query_attr = self.model.value_to_list_on_cache_query()
 118
 119         to_return = None
 120         to_cache = {}
 121
 122         with_aggregates = len(self.query.aggregates) > 0
 123         key_list = self._fetch_from_query_cache(cache_key)
 124
 125         if key_list is None:
 126             if not with_aggregates:
 127                 values_list = [on_cache_query_attr]
 128
 129                 if len(self.query.extra):
 130                     values_list += self.query.extra.keys()
 131
 132                 key_list = [v[0] for v in self.values_list(*values_list)]
 133                 to_cache[cache_key] = (datetime.datetime.now(), key_list)
 134             else:
 135                 to_return = list(super(CachedQuerySet, self).iterator())
 136                 to_cache[cache_key] = (datetime.datetime.now(), [
 137                     (row.__dict__[on_cache_query_attr], dict([(k, row.__dict__[k]) for k in self.query.aggregates.keys()]))
 138                     for row in to_return])
 139         elif with_aggregates:
 140             tmp = key_list
 141             key_list = [k[0] for k in tmp]
 142             with_aggregates = [k[1] for k in tmp]
 143             del tmp
 144
 145         if (not to_return) and key_list:
 146             row_keys = [self.model.infer_cache_key({on_cache_query_attr: attr}) for attr in key_list]
 147             cached = cache.get_many(row_keys)
 148
 149             to_return = [
 150                 (ck in cached) and self.obj_from_datadict(cached[ck]) or ToFetch(force_unicode(key_list[i])) for i, ck in enumerate(row_keys)
 151             ]
 152
 153             if len(cached) != len(row_keys):
 154                 to_fetch = [unicode(tr) for tr in to_return if isinstance(tr, ToFetch)]
 155
 156                 fetched = dict([(force_unicode(r.__dict__[on_cache_query_attr]), r) for r in
 157                               models.query.QuerySet(self.model).filter(**{"%s__in" % on_cache_query_attr: to_fetch})])
 158
 159                 to_return = [(isinstance(tr, ToFetch) and fetched[unicode(tr)] or tr) for tr in to_return]
 160                 to_cache.update(dict([(self.model.infer_cache_key({on_cache_query_attr: attr}), r._as_dict()) for attr, r in fetched.items()]))
 161
 162             if with_aggregates:
 163                 for i, r in enumerate(to_return):
 164                     r.__dict__.update(with_aggregates[i])
 165
 166
 167         if len(to_cache):
 168             cache.set_many(to_cache, 60 * 60)
 169
 170         if to_return:
 171             for row in to_return:
 172                 if hasattr(row, 'leaf'):
 173                     yield row.leaf
 174                 else:
 175                     yield row
 176
 177     def _get_query_hash(self):
 178         return md5(unicode(self.query).encode("utf-8")).hexdigest()
 179
 180
 181
 182 class CachedManager(models.Manager):
 183     use_for_related_fields = True
 184
 185     def get_query_set(self):
 186         return CachedQuerySet(self.model)
 187
 188     def get_or_create(self, *args, **kwargs):
 189         try:
 190             return self.get(*args, **kwargs)
 191         except:
 192             return super(CachedManager, self).get_or_create(*args, **kwargs)
 193
 194
 195 class DenormalizedField(object):
 196     def __init__(self, manager, *args, **kwargs):
 197         self.manager = manager
 198         self.filter = (args, kwargs)
 199
 200     def setup_class(self, cls, name):
 201         dict_name = '_%s_dencache_' % name
 202
 203         def getter(inst):
 204             val = inst.__dict__.get(dict_name, None)
 205
 206             if val is None:
 207                 val = getattr(inst, self.manager).filter(*self.filter[0], **self.filter[1]).count()
 208                 inst.__dict__[dict_name] = val
 209                 inst.cache()
 210
 211             return val
 212
 213         def reset_cache(inst):
 214             inst.__dict__.pop(dict_name, None)
 215             inst.uncache()
 216
 217         cls.add_to_class(name, property(getter))
 218         cls.add_to_class("reset_%s_cache" % name, reset_cache)
 219
 220
 221 class BaseMetaClass(models.Model.__metaclass__):
 222     to_denormalize = []
 223
 224     def __new__(cls, *args, **kwargs):
 225         new_cls = super(BaseMetaClass, cls).__new__(cls, *args, **kwargs)
 226
 227         BaseMetaClass.to_denormalize.extend(
 228             [(new_cls, name, field) for name, field in new_cls.__dict__.items() if isinstance(field, DenormalizedField)]
 229         )
 230
 231         return new_cls
 232
 233     @classmethod
 234     def setup_denormalizes(cls):
 235         for new_cls, name, field in BaseMetaClass.to_denormalize:
 236             field.setup_class(new_cls, name)
 237
 238
 239 class BaseModel(models.Model):
 240     __metaclass__ = BaseMetaClass
 241
 242     objects = CachedManager()
 243
 244     class Meta:
 245         abstract = True
 246         app_label = 'forum'
 247
 248     def __init__(self, *args, **kwargs):
 249         super(BaseModel, self).__init__(*args, **kwargs)
 250         self.reset_original_state(kwargs.keys())
 251
 252     def reset_original_state(self, reset_fields=None):
 253         self._original_state = self._as_dict()
 254
 255         if reset_fields:
 256             self._original_state.update(dict([(f, None) for f in reset_fields]))
 257
 258     def get_dirty_fields(self):
 259         return [f.name for f in self._meta.fields if self._original_state[f.attname] != self.__dict__[f.attname]]
 260
 261     def _as_dict(self):
 262         return dict([(name, getattr(self, name)) for name in
 263                      ([f.attname for f in self._meta.fields] + [k for k in self.__dict__.keys() if k.endswith('_dencache_')])
 264         ])
 265
 266     def _get_update_kwargs(self):
 267         return dict([
 268             (f.name, getattr(self, f.name)) for f in self._meta.fields if self._original_state[f.attname] != self.__dict__[f.attname]
 269         ])
 270
 271     def save(self, full_save=False, *args, **kwargs):
 272         put_back = [k for k, v in self.__dict__.items() if isinstance(v, models.expressions.ExpressionNode)]
 273
 274         if hasattr(self, '_state'):
 275             self._state.db = 'default'
 276
 277         if self.id and not full_save:
 278             self.__class__.objects.filter(id=self.id).update(**self._get_update_kwargs())
 279         else:
 280             super(BaseModel, self).save()
 281
 282         if put_back:
 283             try:
 284                 self.__dict__.update(
 285                     self.__class__.objects.filter(id=self.id).values(*put_back)[0]
 286                 )
 287             except:
 288                 logging.error("Unable to read %s from %s" % (", ".join(put_back), self.__class__.__name__))
 289                 self.uncache()
 290
 291         self.reset_original_state()
 292         self._set_query_cache_invalidation_timestamp()
 293         self.cache()
 294
 295     @classmethod
 296     def _get_cache_query_invalidation_key(cls):
 297         return cls._generate_cache_key("INV_TS")
 298
 299     @classmethod
 300     def _set_query_cache_invalidation_timestamp(cls):
 301         cache.set(cls._get_cache_query_invalidation_key(), datetime.datetime.now(), 60 * 60 * 24)
 302
 303         for base in filter(lambda c: issubclass(c, BaseModel) and (not c is BaseModel), cls.__bases__):
 304             base._set_query_cache_invalidation_timestamp()
 305
 306     @classmethod
 307     def _generate_cache_key(cls, key, group=None):
 308         if group is None:
 309             group = cls.__name__
 310
 311         return '%s:%s:%s' % (settings.APP_URL, group, key)
 312
 313     def cache_key(self):
 314         return self._generate_cache_key(self.id)
 315
 316     @classmethod
 317     def value_to_list_on_cache_query(cls):
 318         return 'id'
 319
 320     @classmethod
 321     def infer_cache_key(cls, querydict):
 322         try:
 323             pk = [v for (k,v) in querydict.items() if k in ('pk', 'pk__exact', 'id', 'id__exact'
 324                             ) or k.endswith('_ptr__pk') or k.endswith('_ptr__id')][0]
 325
 326             return cls._generate_cache_key(pk)
 327         except:
 328             return None
 329
 330     def cache(self):
 331         cache.set(self.cache_key(), self._as_dict(), 60 * 60)
 332
 333     def uncache(self):
 334         cache.delete(self.cache_key())
 335
 336     def delete(self):
 337         self.uncache()
 338         self._set_query_cache_invalidation_timestamp()
 339         super(BaseModel, self).delete()
 340
 341
 342 from user import User
 343 from node import Node, NodeRevision, NodeManager
 344 from action import Action
 345
 346
 347
 348
 349