From ef5d85cc1eae72ca48add569d84e9e75b7b92758 Mon Sep 17 00:00:00 2001 From: hernani Date: Mon, 19 Apr 2010 23:07:14 +0000 Subject: [PATCH] more improvements in cache and denormalized data handling git-svn-id: http://svn.osqa.net/svnroot/osqa/trunk@58 0cfe37f9-358a-4d5e-be75-b63607b5c754 --- forum/badges/base.py | 8 ++-- forum/models/__init__.py | 2 +- forum/models/base.py | 55 +++++++++++++++++++++++--- forum/models/node.py | 10 ++--- forum/models/user.py | 8 ++-- forum/settings/__init__.py | 2 +- forum_modules/default_badges/badges.py | 14 +++---- 7 files changed, 71 insertions(+), 28 deletions(-) diff --git a/forum/badges/base.py b/forum/badges/base.py index c2dbe30..6e5d144 100644 --- a/forum/badges/base.py +++ b/forum/badges/base.py @@ -5,6 +5,7 @@ from django.contrib.contenttypes.models import ContentType from django.db.models.signals import post_save from forum.models.user import activity_record +from forum.models.base import denorm_update from forum.models import Badge, Award, Activity, Node import logging @@ -59,12 +60,11 @@ class AbstractBadge(object): class CountableAbstractBadge(AbstractBadge): def __init__(self, model, field, expected_value, handler): - def wrapper(instance, **kwargs): - dirty_fields = instance.get_dirty_fields() - if field in dirty_fields and instance.__dict__[field] == expected_value: + def wrapper(instance, sfield, old, new, **kwargs): + if sfield == field and (new == expected_value) or (old < expected_value and new > expected_value): handler(instance=instance) - post_save.connect(wrapper, sender=model, weak=False) + denorm_update.connect(wrapper, sender=model, weak=False) class PostCountableAbstractBadge(CountableAbstractBadge): def __init__(self, model, field, expected_value): diff --git a/forum/models/__init__.py b/forum/models/__init__.py index df42a5b..5b6414a 100644 --- a/forum/models/__init__.py +++ b/forum/models/__init__.py @@ -10,7 +10,7 @@ from utils import KeyValue try: from south.modelsinspector import add_introspection_rules - add_introspection_rules([], [r"^forum\.models\.utils\.\w+"]) + add_introspection_rules([], [r"^forum\.models\.\w+\.\w+"]) except: pass diff --git a/forum/models/base.py b/forum/models/base.py index 08ccb44..41f4481 100644 --- a/forum/models/base.py +++ b/forum/models/base.py @@ -1,4 +1,5 @@ import datetime +import re from urllib import quote_plus, urlencode from django.db import models, IntegrityError, connection, transaction from django.utils.http import urlquote as django_urlquote @@ -21,6 +22,15 @@ from forum.const import * class CachedManager(models.Manager): use_for_related_fields = True + int_cache_re = re.compile('^_[\w_]+cache$') + + def cache_obj(self, obj): + int_cache_keys = [k for k in obj.__dict__.keys() if self.int_cache_re.match(k)] + + for k in int_cache_keys: + del obj.__dict__[k] + + cache.set(self.model.cache_key(obj.id), obj, 60 * 60) def get(self, *args, **kwargs): try: @@ -35,7 +45,9 @@ class CachedManager(models.Manager): if obj is None: obj = super(CachedManager, self).get(*args, **kwargs) - cache.set(key, obj, 60 * 60) + self.cache_obj(obj) + else: + d = obj.__dict__ return obj @@ -47,6 +59,17 @@ class CachedManager(models.Manager): except: return super(CachedManager, self).get_or_create(*args, **kwargs) +denorm_update = django.dispatch.Signal(providing_args=["instance", "field", "old", "new"]) + +class DenormalizedField(models.PositiveIntegerField): + __metaclass__ = models.SubfieldBase + + def contribute_to_class(self, cls, name): + super (DenormalizedField, self).contribute_to_class(cls, name) + if not hasattr(cls, '_denormalizad_fields'): + cls._denormalizad_fields = [] + + cls._denormalizad_fields.append(name) class BaseModel(models.Model): objects = CachedManager() @@ -69,9 +92,32 @@ class BaseModel(models.Model): if self._original_state.get(k, missing) == missing or self._original_state[k] != v]) def save(self, *args, **kwargs): + put_back = None + + if hasattr(self.__class__, '_denormalizad_fields'): + dirty = self.get_dirty_fields() + put_back = [f for f in self.__class__._denormalizad_fields if f in dirty] + + if put_back: + for n in put_back: + self.__dict__[n] = models.F(n) + (self.__dict__[n] - dirty[n]) + super(BaseModel, self).save(*args, **kwargs) + + if put_back: + try: + self.__dict__.update( + self.__class__.objects.filter(id=self.id).values(*put_back)[0] + ) + for f in put_back: + denorm_update.send(sender=self.__class__, instance=self, field=f, + old=self._original_state[f], new=self.__dict__[f]) + except: + #todo: log this properly + pass + self._original_state = dict(self.__dict__) - cache.set(self.cache_key(self.pk), self, 86400) + self.__class__.objects.cache_obj(self) def delete(self): cache.delete(self.cache_key(self.pk)) @@ -87,10 +133,7 @@ class UndeletedObjectManager(models.Manager): def get_query_set(self): return super(UndeletedObjectManager, self).get_query_set().filter(deleted=False) -class GenericContent(BaseModel): - """ - Base class for Vote, Comment and FlaggedItem - """ +class GenericContent(models.Model): content_type = models.ForeignKey(ContentType) object_id = models.PositiveIntegerField() content_object = generic.GenericForeignKey('content_type', 'object_id') diff --git a/forum/models/node.py b/forum/models/node.py index 9e734d2..00cbbd0 100644 --- a/forum/models/node.py +++ b/forum/models/node.py @@ -90,12 +90,12 @@ class Node(BaseModel, NodeContent, DeletableContent): tags = models.ManyToManyField('Tag', related_name='%(class)ss') - score = models.IntegerField(default=0) - vote_up_count = models.IntegerField(default=0) + score = DenormalizedField(default=0) + vote_up_count = DenormalizedField(default=0) vote_down_count = models.IntegerField(default=0) - comment_count = models.PositiveIntegerField(default=0) - offensive_flag_count = models.SmallIntegerField(default=0) + comment_count = DenormalizedField(default=0) + offensive_flag_count = DenormalizedField(default=0) last_edited_at = models.DateTimeField(null=True, blank=True) last_edited_by = models.ForeignKey(User, null=True, blank=True, related_name='last_edited_%(class)ss') @@ -205,7 +205,7 @@ class Node(BaseModel, NodeContent, DeletableContent): app_label = 'forum' -class NodeRevision(NodeContent): +class NodeRevision(BaseModel, NodeContent): node = models.ForeignKey(Node, related_name='revisions') summary = models.CharField(max_length=300) revision = models.PositiveIntegerField() diff --git a/forum/models/user.py b/forum/models/user.py index 82cfe3a..eaa1516 100644 --- a/forum/models/user.py +++ b/forum/models/user.py @@ -80,11 +80,11 @@ class User(BaseModel, DjangoUser): is_approved = models.BooleanField(default=False) email_isvalid = models.BooleanField(default=False) email_key = models.CharField(max_length=32, null=True) - reputation = models.PositiveIntegerField(default=1) + reputation = DenormalizedField(default=1) - gold = models.SmallIntegerField(default=0) - silver = models.SmallIntegerField(default=0) - bronze = models.SmallIntegerField(default=0) + gold = DenormalizedField(default=0) + silver = DenormalizedField(default=0) + bronze = DenormalizedField(default=0) questions_per_page = models.SmallIntegerField(choices=QUESTIONS_PER_PAGE_CHOICES, default=10) hide_ignored_questions = models.BooleanField(default=False) diff --git a/forum/settings/__init__.py b/forum/settings/__init__.py index 2446496..5a8b308 100644 --- a/forum/settings/__init__.py +++ b/forum/settings/__init__.py @@ -5,7 +5,7 @@ from forms import ImageFormWidget from django.forms.widgets import Textarea from django.utils.translation import ugettext_lazy as _ -INTERNAL_VERSION = Setting('INTERNAL_VERSION', "56") +INTERNAL_VERSION = Setting('INTERNAL_VERSION', "58") SETTINGS_PACK = Setting('SETTINGS_PACK', "default") from basic import * diff --git a/forum_modules/default_badges/badges.py b/forum_modules/default_badges/badges.py index 057d2ce..eed611c 100644 --- a/forum_modules/default_badges/badges.py +++ b/forum_modules/default_badges/badges.py @@ -7,6 +7,7 @@ from forum.badges.base import PostCountableAbstractBadge, ActivityAbstractBadge, ActivityCountAbstractBadge, CountableAbstractBadge, AbstractBadge, NodeCountableAbstractBadge from forum.models import Node, Question, Answer, Activity, Tag from forum.models.user import activity_record +from forum.models.base import denorm_update from forum import const import settings @@ -237,22 +238,21 @@ class TeacherBadge(CountableAbstractBadge): class AcceptedAndVotedAnswerAbstractBadge(AbstractBadge): def __init__(self, up_votes, handler): def wrapper(sender, instance, **kwargs): - if sender is Node: - if not (instance.node_type == "answer" and "vote_up_count" in instance.get_dirty_fields()): + if sender is Answer: + if (not kwargs['field'] == "score") or (kwargs['new'] < kwargs['old']): return answer = instance.leaf + vote_count = kwargs['new'] else: answer = instance.content_object + vote_count = answer.vote_up_count - accepted = answer.accepted - vote_count = answer.vote_up_count - - if accepted and vote_count == up_votes: + if answer.accepted and vote_count == up_votes: handler(answer) activity_record.connect(wrapper, sender=const.TYPE_ACTIVITY_MARK_ANSWER, weak=False) - post_save.connect(wrapper, sender=Node, weak=False) + denorm_update.connect(wrapper, sender=Node, weak=False) class EnlightenedBadge(AcceptedAndVotedAnswerAbstractBadge): -- 2.45.2