]> git.openstreetmap.org Git - osqa.git/blob - forum/models/base.py
33bf0c8dfbb4afe54496fae6958a62d1b71b46a7
[osqa.git] / forum / models / base.py
1 import datetime
2 import re
3 try:
4     from hashlib import md5
5 except:
6     from md5 import new as md5
7 from urllib import quote_plus, urlencode
8 from django.db import models, IntegrityError, connection, transaction
9 from django.utils.http import urlquote  as django_urlquote
10 from django.utils.html import strip_tags
11 from django.conf import settings as django_settings
12 from django.core.urlresolvers import reverse
13 from django.contrib.contenttypes import generic
14 from django.contrib.contenttypes.models import ContentType
15 from django.core.cache import cache
16 from django.template.defaultfilters import slugify
17 from django.db.models.signals import post_delete, post_save, pre_save, pre_delete
18 from django.utils.translation import ugettext as _
19 from django.utils.safestring import mark_safe
20 from django.utils.encoding import force_unicode
21 from django.contrib.sitemaps import ping_google
22 import django.dispatch
23 from forum import settings
24 import logging
25
26
27 if not hasattr(cache, 'get_many'):
28     #put django 1.2 code here
29     pass
30
31 class LazyQueryList(object):
32     def __init__(self, model, items):
33         self.items = items
34         self.model = model
35
36     def __getitem__(self, k):
37         return self.model.objects.get(id=self.items[k][0])
38
39     def __iter__(self):
40         for id in self.items:
41             yield self.model.objects.get(id=id[0])
42
43     def __len__(self):
44         return len(self.items)
45
46 class ToFetch(unicode):
47     pass
48
49 class CachedQuerySet(models.query.QuerySet):
50
51     def lazy(self):
52         if not len(self.query.aggregates):
53             values_list = ['id']
54
55             if len(self.query.extra):
56                 extra_keys = self.query.extra.keys()
57                 values_list += extra_keys
58
59             return LazyQueryList(self.model, list(self.values_list(*values_list)))
60         else:
61             return self
62
63     def obj_from_datadict(self, datadict):
64         obj = self.model()
65         obj.__dict__.update(datadict)
66
67         if hasattr(obj, '_state'):
68             obj._state.db = 'default'
69
70         return obj
71
72     def _base_clone(self):
73         return self._clone(klass=models.query.QuerySet)
74
75     def get(self, *args, **kwargs):
76         key = self.model.infer_cache_key(kwargs)
77
78         if key is not None:
79             obj = cache.get(key)
80
81             if obj is None:
82                 obj = self._base_clone().get(*args, **kwargs)
83                 obj.cache()
84             else:
85                 obj = self.obj_from_datadict(obj)
86                 obj.reset_original_state()
87
88             return obj
89
90         return self._base_clone().get(*args, **kwargs)
91
92     def _fetch_from_query_cache(self, key):
93         invalidation_key = self.model._get_cache_query_invalidation_key()
94         cached_result = cache.get_many([invalidation_key, key])
95
96         if not invalidation_key in cached_result:
97             self.model._set_query_cache_invalidation_timestamp()
98             return None
99
100         if (key in cached_result) and(cached_result[invalidation_key] < cached_result[key][0]):
101             return cached_result[key][1]
102
103         return None
104
105     def count(self):
106         cache_key = self.model._generate_cache_key("CNT:%s" % self._get_query_hash())
107         result = self._fetch_from_query_cache(cache_key)
108
109         if result is not None:
110             return result
111
112         result = super(CachedQuerySet, self).count()
113         cache.set(cache_key, (datetime.datetime.now(), result), 60 * 60)
114         return result
115
116     def iterator(self):
117         cache_key = self.model._generate_cache_key("QUERY:%s" % self._get_query_hash())
118         on_cache_query_attr = self.model.value_to_list_on_cache_query()
119
120         to_return = None
121         to_cache = {}
122
123         with_aggregates = len(self.query.aggregates) > 0
124         key_list = self._fetch_from_query_cache(cache_key)
125
126         if key_list is None:
127             if not with_aggregates:
128                 values_list = [on_cache_query_attr]
129
130                 if len(self.query.extra):
131                     values_list += self.query.extra.keys()
132
133                 key_list = [v[0] for v in self.values_list(*values_list)]
134                 to_cache[cache_key] = (datetime.datetime.now(), key_list)
135             else:
136                 to_return = list(super(CachedQuerySet, self).iterator())
137                 to_cache[cache_key] = (datetime.datetime.now(), [
138                     (row.__dict__[on_cache_query_attr], dict([(k, row.__dict__[k]) for k in self.query.aggregates.keys()]))
139                     for row in to_return])
140         elif with_aggregates:
141             tmp = key_list
142             key_list = [k[0] for k in tmp]
143             with_aggregates = [k[1] for k in tmp]
144             del tmp
145
146         if (not to_return) and key_list:
147             row_keys = [self.model.infer_cache_key({on_cache_query_attr: attr}) for attr in key_list]
148             cached = cache.get_many(row_keys)
149
150             to_return = [
151                 (ck in cached) and self.obj_from_datadict(cached[ck]) or ToFetch(force_unicode(key_list[i])) for i, ck in enumerate(row_keys)
152             ]
153
154             if len(cached) != len(row_keys):
155                 to_fetch = [unicode(tr) for tr in to_return if isinstance(tr, ToFetch)]
156
157                 fetched = dict([(force_unicode(r.__dict__[on_cache_query_attr]), r) for r in
158                               models.query.QuerySet(self.model).filter(**{"%s__in" % on_cache_query_attr: to_fetch})])
159
160                 to_return = [(isinstance(tr, ToFetch) and fetched[unicode(tr)] or tr) for tr in to_return]
161                 to_cache.update(dict([(self.model.infer_cache_key({on_cache_query_attr: attr}), r._as_dict()) for attr, r in fetched.items()]))
162
163             if with_aggregates:
164                 for i, r in enumerate(to_return):
165                     r.__dict__.update(with_aggregates[i])
166
167
168         if len(to_cache):
169             cache.set_many(to_cache, 60 * 60)
170
171         if to_return:
172             for row in to_return:
173                 if hasattr(row, 'leaf'):
174                     yield row.leaf
175                 else:
176                     yield row
177
178     def _get_query_hash(self):
179         return md5(unicode(self.query).encode("utf-8")).hexdigest()
180
181
182
183 class CachedManager(models.Manager):
184     use_for_related_fields = True
185
186     def get_query_set(self):
187         return CachedQuerySet(self.model)
188
189     def get_or_create(self, *args, **kwargs):
190         try:
191             return self.get(*args, **kwargs)
192         except:
193             return super(CachedManager, self).get_or_create(*args, **kwargs)
194
195
196 class DenormalizedField(object):
197     def __init__(self, manager, *args, **kwargs):
198         self.manager = manager
199         self.filter = (args, kwargs)
200
201     def setup_class(self, cls, name):
202         dict_name = '_%s_dencache_' % name
203
204         def getter(inst):
205             val = inst.__dict__.get(dict_name, None)
206
207             if val is None:
208                 val = getattr(inst, self.manager).filter(*self.filter[0], **self.filter[1]).count()
209                 inst.__dict__[dict_name] = val
210                 inst.cache()
211
212             return val
213
214         def reset_cache(inst):
215             inst.__dict__.pop(dict_name, None)
216             inst.uncache()
217
218         cls.add_to_class(name, property(getter))
219         cls.add_to_class("reset_%s_cache" % name, reset_cache)
220
221
222 class BaseMetaClass(models.Model.__metaclass__):
223     to_denormalize = []
224
225     def __new__(cls, *args, **kwargs):
226         new_cls = super(BaseMetaClass, cls).__new__(cls, *args, **kwargs)
227
228         BaseMetaClass.to_denormalize.extend(
229             [(new_cls, name, field) for name, field in new_cls.__dict__.items() if isinstance(field, DenormalizedField)]
230         )
231
232         return new_cls
233
234     @classmethod
235     def setup_denormalizes(cls):
236         for new_cls, name, field in BaseMetaClass.to_denormalize:
237             field.setup_class(new_cls, name)
238
239
240 class BaseModel(models.Model):
241     __metaclass__ = BaseMetaClass
242
243     objects = CachedManager()
244
245     class Meta:
246         abstract = True
247         app_label = 'forum'
248
249     def __init__(self, *args, **kwargs):
250         super(BaseModel, self).__init__(*args, **kwargs)
251         self.reset_original_state(kwargs.keys())
252
253     def reset_original_state(self, reset_fields=None):
254         self._original_state = self._as_dict()
255         
256         if reset_fields:
257             self._original_state.update(dict([(f, None) for f in reset_fields]))
258
259     def get_dirty_fields(self):
260         return [f.name for f in self._meta.fields if self._original_state[f.attname] != self.__dict__[f.attname]]
261
262     def _as_dict(self):
263         return dict([(name, getattr(self, name)) for name in
264                      ([f.attname for f in self._meta.fields] + [k for k in self.__dict__.keys() if k.endswith('_dencache_')])
265         ])
266
267     def _get_update_kwargs(self):
268         return dict([
269             (f.name, getattr(self, f.name)) for f in self._meta.fields if self._original_state[f.attname] != self.__dict__[f.attname]
270         ])
271
272     def save(self, full_save=False, *args, **kwargs):
273         put_back = [k for k, v in self.__dict__.items() if isinstance(v, models.expressions.ExpressionNode)]
274
275         if hasattr(self, '_state'):
276             self._state.db = 'default'
277
278         if self.id and not full_save:
279             self.__class__.objects.filter(id=self.id).update(**self._get_update_kwargs())
280         else:
281             super(BaseModel, self).save()
282
283         if put_back:
284             try:
285                 self.__dict__.update(
286                     self.__class__.objects.filter(id=self.id).values(*put_back)[0]
287                 )
288             except:
289                 logging.error("Unable to read %s from %s" % (", ".join(put_back), self.__class__.__name__))
290                 self.uncache()
291
292         self.reset_original_state()
293         self._set_query_cache_invalidation_timestamp()
294         self.cache()
295
296     @classmethod
297     def _get_cache_query_invalidation_key(cls):
298         return cls._generate_cache_key("INV_TS")
299
300     @classmethod
301     def _set_query_cache_invalidation_timestamp(cls):
302         cache.set(cls._get_cache_query_invalidation_key(), datetime.datetime.now(), 60 * 60 * 24)
303
304         for base in filter(lambda c: issubclass(c, BaseModel) and (not c is BaseModel), cls.__bases__):
305             base._set_query_cache_invalidation_timestamp()
306
307     @classmethod
308     def _generate_cache_key(cls, key, group=None):
309         if group is None:
310             group = cls.__name__
311
312         return '%s:%s:%s' % (settings.APP_URL, group, key)
313
314     def cache_key(self):
315         return self._generate_cache_key(self.id)
316
317     @classmethod
318     def value_to_list_on_cache_query(cls):
319         return 'id'
320
321     @classmethod
322     def infer_cache_key(cls, querydict):
323         try:
324             pk = [v for (k,v) in querydict.items() if k in ('pk', 'pk__exact', 'id', 'id__exact'
325                             ) or k.endswith('_ptr__pk') or k.endswith('_ptr__id')][0]
326
327             cache_key = cls._generate_cache_key(pk)
328
329             if len(cache_key) > django_settings.CACHE_MAX_KEY_LENGTH:
330                 cache_key = cache_key[:django_settings.CACHE_MAX_KEY_LENGTH]
331
332             return cache_key
333         except:
334             return None
335
336     def cache(self):
337         cache.set(self.cache_key(), self._as_dict(), 60 * 60)
338
339     def uncache(self):
340         cache.delete(self.cache_key())
341
342     def delete(self):
343         self.uncache()
344         self._set_query_cache_invalidation_timestamp()
345         super(BaseModel, self).delete()
346
347
348 from user import User
349 from node import Node, NodeRevision, NodeManager
350 from action import Action
351
352
353
354
355