forum_modules/sximporter/importer.py

   1 # -*- coding: utf-8 -*-
   2
   3 from xml.dom import minidom
   4 from datetime import datetime, timedelta
   5 import time
   6 import re
   7 import os
   8 import gc
   9 from django.utils.translation import ugettext as _
  10 from django.template.defaultfilters import slugify
  11 from forum.models.utils import dbsafe_encode
  12 from orm import orm
  13
  14 from django.utils.encoding import force_unicode
  15
  16 try:
  17     from cPickle import loads, dumps
  18 except ImportError:
  19     from pickle import loads, dumps
  20
  21 from copy import deepcopy
  22 from base64 import b64encode, b64decode
  23 from zlib import compress, decompress
  24
  25 from xml.sax import make_parser
  26 from xml.sax.handler import ContentHandler
  27
  28 class SXTableHandler(ContentHandler):
  29     def __init__(self, fname, callback):
  30         self.in_row = False
  31         self.el_data = {}
  32         self.ch_data = ''
  33
  34         self.fname = fname.lower()
  35         self.callback = callback
  36
  37     def startElement(self, name, attrs):
  38         if name.lower() == self.fname:
  39             pass
  40         elif name.lower() == "row":
  41             self.in_row = True
  42
  43     def characters(self, ch):
  44         self.ch_data += ch
  45
  46     def endElement(self, name):
  47         if name.lower() == self.fname:
  48             pass
  49         elif name.lower() == "row":
  50             self.callback(self.el_data)
  51
  52             self.in_row = False
  53             del self.el_data
  54             self.el_data = {}
  55         elif self.in_row:
  56             self.el_data[name.lower()] = self.ch_data.strip()
  57             del self.ch_data
  58             self.ch_data = ''
  59
  60
  61 def readTable(path, name, callback):
  62     parser = make_parser()
  63     handler = SXTableHandler(name, callback)
  64     parser.setContentHandler(handler)
  65
  66     f = os.path.join(path, "%s.xml" % name)
  67     parser.parse(f)
  68
  69
  70 def dbsafe_encode(value):
  71     return force_unicode(b64encode(compress(dumps(deepcopy(value)))))
  72
  73 def getText(el):
  74     rc = ""
  75     for node in el.childNodes:
  76         if node.nodeType == node.TEXT_NODE:
  77             rc = rc + node.data
  78     return rc.strip()
  79
  80 msstrip = re.compile(r'^(.*)\.\d+')
  81 def readTime(ts):
  82     noms = msstrip.match(ts)
  83     if noms:
  84         ts = noms.group(1)
  85
  86     return datetime(*time.strptime(ts, '%Y-%m-%dT%H:%M:%S')[0:6])
  87
  88 #def readEl(el):
  89 #    return dict([(n.tagName.lower(), getText(n)) for n in el.childNodes if n.nodeType == el.ELEMENT_NODE])
  90
  91 #def readTable(dump, name):
  92 #    for e in minidom.parseString(dump.read("%s.xml" % name)).getElementsByTagName('row'):
  93 #        yield readEl(e)
  94 #return [readEl(e) for e in minidom.parseString(dump.read("%s.xml" % name)).getElementsByTagName('row')]
  95
  96 google_accounts_lookup = re.compile(r'^https?://www.google.com/accounts/')
  97 yahoo_accounts_lookup = re.compile(r'^https?://me.yahoo.com/a/')
  98
  99 openid_lookups = [
 100         re.compile(r'^https?://www.google.com/profiles/(?P<uname>\w+(\.\w+)*)/?$'),
 101         re.compile(r'^https?://me.yahoo.com/(?P<uname>\w+(\.\w+)*)/?$'),
 102         re.compile(r'^https?://openid.aol.com/(?P<uname>\w+(\.\w+)*)/?$'),
 103         re.compile(r'^https?://(?P<uname>\w+(\.\w+)*).myopenid.com/?$'),
 104         re.compile(r'^https?://flickr.com/(\w+/)*(?P<uname>\w+(\.\w+)*)/?$'),
 105         re.compile(r'^https?://technorati.com/people/technorati/(?P<uname>\w+(\.\w+)*)/?$'),
 106         re.compile(r'^https?://(?P<uname>\w+(\.\w+)*).wordpress.com/?$'),
 107         re.compile(r'^https?://(?P<uname>\w+(\.\w+)*).blogspot.com/?$'),
 108         re.compile(r'^https?://(?P<uname>\w+(\.\w+)*).livejournal.com/?$'),
 109         re.compile(r'^https?://claimid.com/(?P<uname>\w+(\.\w+)*)/?$'),
 110         re.compile(r'^https?://(?P<uname>\w+(\.\w+)*).pip.verisignlabs.com/?$'),
 111         re.compile(r'^https?://getopenid.com/(?P<uname>\w+(\.\w+)*)/?$'),
 112         re.compile(r'^https?://[\w\.]+/(\w+/)*(?P<uname>\w+(\.\w+)*)/?$'),
 113         re.compile(r'^https?://(?P<uname>[\w\.]+)/?$'),
 114         ]
 115
 116 def final_username_attempt(sxu):
 117     openid = sxu.get('openid', None)
 118
 119     if openid:
 120         if google_accounts_lookup.search(openid):
 121             return UnknownGoogleUser(sxu.get('id'))
 122         if yahoo_accounts_lookup.search(openid):
 123             return UnknownYahooUser(sxu.get('id'))
 124
 125         for lookup in openid_lookups:
 126             if lookup.search(openid):
 127                 return lookup.search(openid).group('uname')
 128
 129     return UnknownUser(sxu.get('id'))
 130
 131 class UnknownUser(object):
 132     def __init__(self, id):
 133         self._id = id
 134
 135     def __str__(self):
 136         return _("user-%(id)s") % {'id': self._id}
 137
 138     def __unicode__(self):
 139         return self.__str__()
 140
 141     def encode(self, *args):
 142         return self.__str__()
 143
 144 class UnknownGoogleUser(UnknownUser):
 145     def __str__(self):
 146         return _("user-%(id)s (google)") % {'id': self._id}
 147
 148 class UnknownYahooUser(UnknownUser):
 149     def __str__(self):
 150         return _("user-%(id)s (yahoo)") % {'id': self._id}
 151
 152
 153 class IdMapper(dict):
 154     def __getitem__(self, key):
 155         key = int(key)
 156         return super(IdMapper, self).get(key, 1)
 157
 158     def __setitem__(self, key, value):
 159         super(IdMapper, self).__setitem__(int(key), int(value))
 160
 161 class IdIncrementer():
 162     def __init__(self, initial):
 163         self.value = initial
 164
 165     def inc(self):
 166         self.value += 1
 167
 168 openidre = re.compile('^https?\:\/\/')
 169 def userimport(path, options):
 170 #users = readTable(dump, "Users")
 171
 172     usernames = []
 173     uidmapper = IdMapper()
 174     #merged_users = []
 175
 176     owneruid = options.get('owneruid', None)
 177     #check for empty values
 178     if not owneruid:
 179         owneruid = None
 180
 181     def callback(sxu):
 182         create = True
 183
 184         if sxu.get('id') == '-1':
 185             return
 186         #print "\n".join(["%s : %s" % i for i in sxu.items()])
 187         if int(sxu.get('id')) == int(owneruid):
 188             osqau = orm.User.objects.get(id=1)
 189             uidmapper[owneruid] = 1
 190             uidmapper[-1] = 1
 191             create = False
 192         else:
 193             username = unicode(sxu.get('displayname',
 194                                sxu.get('displaynamecleaned', sxu.get('realname', final_username_attempt(sxu)))))[:30]
 195
 196             if username in usernames:
 197             #if options.get('mergesimilar', False) and sxu.get('email', 'INVALID') == user_by_name[username].email:
 198             #    osqau = user_by_name[username]
 199             #    create = False
 200             #    uidmapper[sxu.get('id')] = osqau.id
 201             #else:
 202                 inc = 0
 203
 204                 while True:
 205                     inc += 1
 206                     totest = "%s %d" % (username[:29 - len(str(inc))], inc)
 207
 208                     if not totest in usernames:
 209                         username = totest
 210                         break
 211
 212         sxbadges = sxu.get('badgesummary', None)
 213         badges = {'1':'0', '2':'0', '3':'0'}
 214
 215         if sxbadges:
 216             badges.update(dict([b.split('=') for b in sxbadges.split()]))
 217
 218         if create:
 219             osqau = orm.User(
 220                     id           = sxu.get('id'),
 221                     username     = username,
 222                     password     = '!',
 223                     email        = sxu.get('email', ''),
 224                     is_superuser = sxu.get('usertypeid') == '5',
 225                     is_staff     = sxu.get('usertypeid') == '4',
 226                     is_active    = True,
 227                     date_joined  = readTime(sxu.get('creationdate')),
 228                     last_seen    = readTime(sxu.get('lastaccessdate')),
 229                     about         = sxu.get('aboutme', ''),
 230                     date_of_birth = sxu.get('birthday', None) and readTime(sxu['birthday']) or None,
 231                     email_isvalid = int(sxu.get('usertypeid')) > 2,
 232                     website       = sxu.get('websiteurl', ''),
 233                     reputation    = int(sxu.get('reputation')),
 234                     gold          = int(badges['1']),
 235                     silver        = int(badges['2']),
 236                     bronze        = int(badges['3']),
 237                     real_name     = sxu.get('realname', '')[:30],
 238                     location      = sxu.get('location', ''),
 239                     )
 240
 241             osqau.save()
 242
 243             user_joins = orm.Action(
 244                     action_type = "userjoins",
 245                     action_date = osqau.date_joined,
 246                     user = osqau
 247                     )
 248             user_joins.save()
 249
 250             rep = orm.ActionRepute(
 251                     value = 1,
 252                     user = osqau,
 253                     date = osqau.date_joined,
 254                     action = user_joins
 255                     )
 256             rep.save()
 257
 258             try:
 259                 orm.SubscriptionSettings.objects.get(user=osqau)
 260             except:
 261                 s = orm.SubscriptionSettings(user=osqau)
 262                 s.save()
 263
 264             uidmapper[osqau.id] = osqau.id
 265         else:
 266             new_about = sxu.get('aboutme', None)
 267             if new_about and osqau.about != new_about:
 268                 if osqau.about:
 269                     osqau.about = "%s\n|\n%s" % (osqau.about, new_about)
 270                 else:
 271                     osqau.about = new_about
 272
 273             osqau.username = sxu.get('displayname',
 274                                      sxu.get('displaynamecleaned', sxu.get('realname', final_username_attempt(sxu))))
 275             osqau.email = sxu.get('email', '')
 276             osqau.reputation += int(sxu.get('reputation'))
 277             osqau.gold += int(badges['1'])
 278             osqau.silver += int(badges['2'])
 279             osqau.bronze += int(badges['3'])
 280
 281             osqau.date_joined = readTime(sxu.get('creationdate'))
 282             osqau.website = sxu.get('websiteurl', '')
 283             osqau.date_of_birth = sxu.get('birthday', None) and readTime(sxu['birthday']) or None
 284             osqau.location = sxu.get('location', '')
 285             osqau.real_name = sxu.get('realname', '')
 286
 287             #merged_users.append(osqau.id)
 288             osqau.save()
 289
 290         usernames.append(osqau.username)
 291
 292         openid = sxu.get('openid', None)
 293         if openid and openidre.match(openid):
 294             assoc = orm.AuthKeyUserAssociation(user=osqau, key=openid, provider="openidurl")
 295             assoc.save()
 296
 297     readTable(path, "Users", callback)
 298
 299     if uidmapper[-1] == -1:
 300         uidmapper[-1] = 1
 301
 302     return uidmapper
 303
 304 def tagsimport(dump, uidmap):
 305 #tags = readTable(dump, "Tags")
 306
 307     tagmap = {}
 308
 309     def callback(sxtag):
 310         otag = orm.Tag(
 311                 id = int(sxtag['id']),
 312                 name = sxtag['name'],
 313                 used_count = int(sxtag['count']),
 314                 created_by_id = uidmap[sxtag.get('userid', 1)],
 315                 )
 316         otag.save()
 317
 318         tagmap[otag.name] = otag
 319
 320     readTable(dump, "Tags", callback)
 321
 322     return tagmap
 323
 324 def add_post_state(name, post, action):
 325     if not "(%s)" % name in post.state_string:
 326         post.state_string = "%s(%s)" % (post.state_string, name)
 327         post.save()
 328
 329     try:
 330         state = orm.NodeState.objects.get(node=post, state_type=name)
 331         state.action = action
 332         state.save()
 333     except:
 334         state = orm.NodeState(node=post, state_type=name, action=action)
 335         state.save()
 336
 337 def remove_post_state(name, post):
 338     if "(%s)" % name in post.state_string:
 339         try:
 340             state = orm.NodeState.objects.get(state_type=name, post=post)
 341             state.delete()
 342         except:
 343             pass
 344     post.state_string = "".join("(%s)" % s for s in re.findall('\w+', post.state_string) if s != name)
 345
 346 def postimport(dump, uidmap, tagmap):
 347 #history = {}
 348 #accepted = {}
 349     all = []
 350
 351     #for h in readTable(dump, "PostHistory"):
 352     #    if not history.get(h.get('postid'), None):
 353     #        history[h.get('postid')] = []
 354     #
 355     #    history[h.get('postid')].append(h)
 356
 357     #posts = readTable(dump, "Posts")
 358
 359     def callback(sxpost):
 360         nodetype = (sxpost.get('posttypeid') == '1') and "nodetype" or "answer"
 361
 362         post = orm.Node(
 363                 node_type = nodetype,
 364                 id = sxpost['id'],
 365                 added_at = readTime(sxpost['creationdate']),
 366                 body = sxpost['body'],
 367                 score = sxpost.get('score', 0),
 368                 author_id = sxpost.get('deletiondate', None) and 1 or uidmap[sxpost.get('owneruserid', 1)]
 369                 )
 370
 371         post.save()
 372
 373         create_action = orm.Action(
 374                 action_type = (nodetype == "nodetype") and "ask" or "answer",
 375                 user_id = post.author_id,
 376                 node = post,
 377                 action_date = post.added_at
 378                 )
 379
 380         create_action.save()
 381
 382         if sxpost.get('lasteditoruserid', None):
 383             revise_action = orm.Action(
 384                     action_type = "revise",
 385                     user_id = uidmap[sxpost.get('lasteditoruserid')],
 386                     node = post,
 387                     action_date = readTime(sxpost['lasteditdate']),
 388                     )
 389
 390             revise_action.save()
 391             post.last_edited = revise_action
 392
 393         if sxpost.get('communityowneddate', None):
 394             wikify_action = orm.Action(
 395                     action_type = "wikify",
 396                     user_id = 1,
 397                     node = post,
 398                     action_date = readTime(sxpost['communityowneddate'])
 399                     )
 400
 401             wikify_action.save()
 402             add_post_state("wiki", post, wikify_action)
 403
 404         if sxpost.get('lastactivityuserid', None):
 405             post.last_activity_by_id = uidmap[sxpost['lastactivityuserid']]
 406             post.last_activity_at = readTime(sxpost['lastactivitydate'])
 407
 408         if sxpost.get('posttypeid') == '1': #question
 409             post.node_type = "question"
 410             post.title = sxpost['title']
 411
 412             tagnames = sxpost['tags'].replace(u'ö', '-').replace(u'é', '').replace(u'à', '')
 413             post.tagnames = tagnames
 414
 415             post.extra_count = sxpost.get('viewcount', 0)
 416
 417             add_tags_to_post(post, tagmap)
 418
 419         else:
 420             post.parent_id = sxpost['parentid']
 421
 422         post.save()
 423
 424         all.append(int(post.id))
 425
 426         del post
 427
 428     readTable(dump, "Posts", callback)
 429
 430     return all
 431
 432 def comment_import(dump, uidmap, posts):
 433 #comments = readTable(dump, "PostComments")
 434     currid = IdIncrementer(max(posts))
 435     mapping = {}
 436
 437     def callback(sxc):
 438         currid.inc()
 439         oc = orm.Node(
 440                 id = currid.value,
 441                 node_type = "comment",
 442                 added_at = readTime(sxc['creationdate']),
 443                 author_id = uidmap[sxc.get('userid', 1)],
 444                 body = sxc['text'],
 445                 parent_id = sxc.get('postid'),
 446                 )
 447
 448         if sxc.get('deletiondate', None):
 449             delete_action = orm.Action(
 450                     action_type = "delete",
 451                     user_id = uidmap[sxc['deletionuserid']],
 452                     action_date = readTime(sxc['deletiondate'])
 453                     )
 454
 455             oc.author_id = uidmap[sxc['deletionuserid']]
 456             oc.save()
 457
 458             delete_action.node = oc
 459             delete_action.save()
 460
 461             add_post_state("deleted", oc, delete_action)
 462         else:
 463             oc.author_id = uidmap[sxc.get('userid', 1)]
 464             oc.save()
 465
 466         create_action = orm.Action(
 467                 action_type = "comment",
 468                 user_id = oc.author_id,
 469                 node = oc,
 470                 action_date = oc.added_at
 471                 )
 472
 473         create_action.save()
 474         oc.save()
 475
 476         posts.append(int(oc.id))
 477         mapping[int(sxc['id'])] = int(oc.id)
 478
 479     readTable(dump, "PostComments", callback)
 480     return posts, mapping
 481
 482
 483 def add_tags_to_post(post, tagmap):
 484     tags = [tag for tag in [tagmap.get(name.strip()) for name in post.tagnames.split(u' ') if name] if tag]
 485     post.tagnames = " ".join([t.name for t in tags]).strip()
 486     post.tags = tags
 487     create_and_activate_revision(post)
 488
 489
 490 def create_and_activate_revision(post):
 491     rev = orm.NodeRevision(
 492             author_id = post.author_id,
 493             body = post.body,
 494             node_id = post.id,
 495             revised_at = post.added_at,
 496             revision = 1,
 497             summary = 'Initial revision',
 498             tagnames = post.tagnames,
 499             title = post.title,
 500             )
 501
 502     rev.save()
 503     post.active_revision_id = rev.id
 504     post.save()
 505
 506 def post_vote_import(dump, uidmap, posts):
 507 #votes = readTable(dump, "Posts2Votes")
 508     close_reasons = {}
 509
 510     def close_callback(r):
 511         close_reasons[r['id']] = r['name']
 512
 513     readTable(dump, "CloseReasons", close_callback)
 514
 515     user2vote = []
 516
 517     def callback(sxv):
 518         action = orm.Action(
 519                 user_id=uidmap[sxv['userid']],
 520                 action_date = readTime(sxv['creationdate']),
 521                 )
 522
 523         if not int(sxv['postid']) in posts: return
 524         node = orm.Node.objects.get(id=sxv['postid'])
 525         action.node = node
 526
 527         if sxv['votetypeid'] == '1':
 528             answer = node
 529             question = orm.Node.objects.get(id=answer.parent_id)
 530
 531             action.action_type = "acceptanswer"
 532             action.save()
 533
 534             answer.marked = True
 535
 536             question.extra_ref_id = answer.id
 537
 538             answer.save()
 539             question.save()
 540
 541         elif sxv['votetypeid'] in ('2', '3'):
 542             if not (action.node.id, action.user_id) in user2vote:
 543                 user2vote.append((action.node.id, action.user_id))
 544
 545                 action.action_type = (sxv['votetypeid'] == '2') and "voteup" or "votedown"
 546                 action.save()
 547
 548                 ov = orm.Vote(
 549                         node_id = action.node.id,
 550                         user_id = action.user_id,
 551                         voted_at = action.action_date,
 552                         value = sxv['votetypeid'] == '2' and 1 or -1,
 553                         action = action
 554                         )
 555                 ov.save()
 556             else:
 557                 action.action_type = "unknown"
 558                 action.save()
 559
 560         elif sxv['votetypeid'] in ('4', '12', '13'):
 561             action.action_type = "flag"
 562             action.save()
 563
 564             of = orm.Flag(
 565                     node = action.node,
 566                     user_id = action.user_id,
 567                     flagged_at = action.action_date,
 568                     reason = '',
 569                     action = action
 570                     )
 571
 572             of.save()
 573
 574         elif sxv['votetypeid'] == '5':
 575             action.action_type = "favorite"
 576             action.save()
 577
 578         elif sxv['votetypeid'] == '6':
 579             action.action_type = "close"
 580             action.extra = dbsafe_encode(close_reasons[sxv['comment']])
 581             action.save()
 582
 583             node.marked = True
 584             node.save()
 585
 586         elif sxv['votetypeid'] == '7':
 587             action.action_type = "unknown"
 588             action.save()
 589
 590             node.marked = False
 591             node.save()
 592
 593             remove_post_state("closed", node)
 594
 595         elif sxv['votetypeid'] == '10':
 596             action.action_type = "delete"
 597             action.save()
 598
 599         elif sxv['votetypeid'] == '11':
 600             action.action_type = "unknown"
 601             action.save()
 602
 603             remove_post_state("deleted", node)
 604
 605         else:
 606             action.action_type = "unknown"
 607             action.save()
 608
 609         if sxv.get('targetrepchange', None):
 610             rep = orm.ActionRepute(
 611                     action = action,
 612                     date = action.action_date,
 613                     user_id = uidmap[sxv['targetuserid']],
 614                     value = int(sxv['targetrepchange'])
 615                     )
 616
 617             rep.save()
 618
 619         if sxv.get('voterrepchange', None):
 620             rep = orm.ActionRepute(
 621                     action = action,
 622                     date = action.action_date,
 623                     user_id = uidmap[sxv['userid']],
 624                     value = int(sxv['voterrepchange'])
 625                     )
 626
 627             rep.save()
 628
 629         if action.action_type in ("acceptanswer", "delete", "close"):
 630             state = {"acceptanswer": "accepted", "delete": "deleted", "close": "closed"}[action.action_type]
 631             add_post_state(state, node, action)
 632
 633     readTable(dump, "Posts2Votes", callback)
 634
 635
 636 def comment_vote_import(dump, uidmap, comments):
 637 #votes = readTable(dump, "Comments2Votes")
 638     user2vote = []
 639     comments2score = {}
 640
 641     def callback(sxv):
 642         if sxv['votetypeid'] == "2":
 643             comment_id = comments[int(sxv['postcommentid'])]
 644             user_id = uidmap[sxv['userid']]
 645
 646             if not (comment_id, user_id) in user2vote:
 647                 user2vote.append((comment_id, user_id))
 648
 649                 action = orm.Action(
 650                         action_type = "voteupcomment",
 651                         user_id = user_id,
 652                         action_date = readTime(sxv['creationdate']),
 653                         node_id = comment_id
 654                         )
 655                 action.save()
 656
 657                 ov = orm.Vote(
 658                         node_id = comment_id,
 659                         user_id = user_id,
 660                         voted_at = action.action_date,
 661                         value = 1,
 662                         action = action
 663                         )
 664
 665                 ov.save()
 666
 667                 if not comment_id in comments2score:
 668                     comments2score[comment_id] = 1
 669                 else:
 670                     comments2score[comment_id] += 1
 671
 672     readTable(dump, "Comments2Votes", callback)
 673
 674     for cid, score in comments2score.items():
 675         orm.Node.objects.filter(id=cid).update(score=score)
 676
 677
 678 def badges_import(dump, uidmap, post_list):
 679 #node_ctype = orm['contenttypes.contenttype'].objects.get(name='node')
 680
 681     sxbadges = {}
 682
 683     def sxcallback(b):
 684         sxbadges[int(b['id'])] = b
 685
 686     readTable(dump, "Badges", sxcallback)
 687
 688     obadges = dict([(b.cls, b) for b in orm.Badge.objects.all()])
 689     user_badge_count = {}
 690
 691     sx_to_osqa = {}
 692
 693     for id, sxb in sxbadges.items():
 694         cls = "".join(sxb['name'].replace('&', 'And').split(' '))
 695
 696         if cls in obadges:
 697             sx_to_osqa[id] = obadges[cls]
 698         else:
 699             osqab = orm.Badge(
 700                     cls = cls,
 701                     awarded_count = 0,
 702                     type = sxb['class']
 703                     )
 704             osqab.save()
 705             sx_to_osqa[id] = osqab
 706
 707     osqaawards = []
 708
 709     def callback(sxa):
 710         badge = sx_to_osqa[int(sxa['badgeid'])]
 711
 712         user_id = uidmap[sxa['userid']]
 713         if not user_badge_count.get(user_id, None):
 714             user_badge_count[user_id] = 0
 715
 716         action = orm.Action(
 717                 action_type = "award",
 718                 user_id = user_id,
 719                 action_date = readTime(sxa['date'])
 720                 )
 721
 722         action.save()
 723
 724         osqaa = orm.Award(
 725                 user_id = uidmap[sxa['userid']],
 726                 badge = badge,
 727                 node_id = post_list[user_badge_count[user_id]],
 728                 awarded_at = action.action_date,
 729                 action = action
 730                 )
 731
 732         osqaa.save()
 733         badge.awarded_count += 1
 734         user_badge_count[user_id] += 1
 735
 736     readTable(dump, "Users2Badges", callback)
 737
 738     for badge in obadges.values():
 739         badge.save()
 740
 741 def pages_import(dump, currid):
 742     currid = IdIncrementer(currid)
 743     registry = {}
 744     #sx_pages = readTable(dump, "FlatPages")
 745
 746     def callback(sxp):
 747         currid.inc()
 748         page = orm.Node(
 749                 id = currid.value,
 750                 node_type = "page",
 751                 title = sxp['name'],
 752                 body = b64decode(sxp['value']),
 753                 extra = dbsafe_encode({
 754                 'path': sxp['url'][1:],
 755                 'mimetype': sxp['contenttype'],
 756                 'template': (sxp['usemaster'] == "true") and "default" or "none",
 757                 'render': "html",
 758                 'sidebar': "",
 759                 'sidebar_wrap': True,
 760                 'sidebar_render': "html",
 761                 'comments': False
 762                 }),
 763                 author_id = 1
 764                 )
 765
 766         page.save()
 767         registry[sxp['url'][1:]] = page.id
 768
 769         create_action = orm.Action(
 770                 action_type = "newpage",
 771                 user_id = page.author_id,
 772                 node = page
 773                 )
 774
 775         create_action.save()
 776
 777         if sxp['active'] == "true" and sxp['contenttype'] == "text/html":
 778             pub_action = orm.Action(
 779                     action_type = "publish",
 780                     user_id = page.author_id,
 781                     node = page
 782                     )
 783
 784             pub_action.save()
 785             add_post_state("published", page, pub_action)
 786
 787     readTable(dump, "FlatPages", callback)
 788
 789     kv = orm.KeyValue(key='STATIC_PAGE_REGISTRY', value=dbsafe_encode(registry))
 790     kv.save()
 791
 792 sx2osqa_set_map = {
 793 u'theme.html.name': 'APP_TITLE',
 794 u'theme.html.footer': 'CUSTOM_FOOTER',
 795 u'theme.html.sidebar': 'SIDEBAR_UPPER_TEXT',
 796 u'theme.html.sidebar-low': 'SIDEBAR_LOWER_TEXT',
 797 u'theme.html.welcome': 'APP_INTRO',
 798 u'theme.html.head': 'CUSTOM_HEAD',
 799 u'theme.html.header': 'CUSTOM_HEADER',
 800 u'theme.css': 'CUSTOM_CSS',
 801 }
 802
 803 html_codes = (
 804 ('&amp;', '&'),
 805 ('&lt;', '<'),
 806 ('&gt;', '>'),
 807 ('&quot;', '"'),
 808 ('&#39;', "'"),
 809 )
 810
 811 def html_decode(html):
 812     html = force_unicode(html)
 813
 814     for args in html_codes:
 815         html = html.replace(*args)
 816
 817     return html
 818
 819
 820 def static_import(dump):
 821 #sx_sets = readTable(dump, "ThemeTextResources")
 822     sx_unknown = {}
 823
 824     def callback(set):
 825         if unicode(set['name']) in sx2osqa_set_map:
 826             try:
 827                 kv = orm.KeyValue.objects.get(key=sx2osqa_set_map[set['name']])
 828                 kv.value = dbsafe_encode(html_decode(set['value']))
 829             except:
 830                 kv = orm.KeyValue(
 831                         key = sx2osqa_set_map[set['name']],
 832                         value = dbsafe_encode(html_decode(set['value']))
 833                         )
 834
 835             kv.save()
 836         else:
 837             sx_unknown[set['name']] = html_decode(set['value'])
 838
 839     readTable(dump, "ThemeTextResources", callback)
 840
 841     unknown = orm.KeyValue(key='SXIMPORT_UNKNOWN_SETS', value=dbsafe_encode(sx_unknown))
 842     unknown.save()
 843
 844 def disable_triggers():
 845     from south.db import db
 846     if db.backend_name == "postgres":
 847         db.execute_many(PG_DISABLE_TRIGGERS)
 848         db.commit_transaction()
 849         db.start_transaction()
 850
 851 def enable_triggers():
 852     from south.db import db
 853     if db.backend_name == "postgres":
 854         db.start_transaction()
 855         db.execute_many(PG_ENABLE_TRIGGERS)
 856         db.commit_transaction()
 857
 858 def reset_sequences():
 859     from south.db import db
 860     if db.backend_name == "postgres":
 861         db.start_transaction()
 862         db.execute_many(PG_SEQUENCE_RESETS)
 863         db.commit_transaction()
 864
 865 def reindex_fts():
 866     from south.db import db
 867     if db.backend_name == "postgres":
 868         db.start_transaction()
 869         db.execute_many("UPDATE forum_noderevision set id = id WHERE TRUE;")
 870         db.commit_transaction()
 871
 872
 873 def sximport(dump, options):
 874     try:
 875         disable_triggers()
 876         triggers_disabled = True
 877     except:
 878         triggers_disabled = False
 879
 880     uidmap = userimport(dump, options)
 881     tagmap = tagsimport(dump, uidmap)
 882     gc.collect()
 883
 884     posts = postimport(dump, uidmap, tagmap)
 885     gc.collect()
 886
 887     posts, comments = comment_import(dump, uidmap, posts)
 888     gc.collect()
 889
 890     post_vote_import(dump, uidmap, posts)
 891     gc.collect()
 892
 893     comment_vote_import(dump, uidmap, comments)
 894     gc.collect()
 895
 896     badges_import(dump, uidmap, posts)
 897
 898     pages_import(dump, max(posts))
 899     static_import(dump)
 900     gc.collect()
 901
 902     from south.db import db
 903     db.commit_transaction()
 904
 905     reset_sequences()
 906
 907     if triggers_disabled:
 908         enable_triggers()
 909         reindex_fts()
 910
 911
 912 PG_DISABLE_TRIGGERS = """
 913 ALTER table auth_user DISABLE TRIGGER ALL;
 914 ALTER table auth_user_groups DISABLE TRIGGER ALL;
 915 ALTER table auth_user_user_permissions DISABLE TRIGGER ALL;
 916 ALTER table forum_keyvalue DISABLE TRIGGER ALL;
 917 ALTER table forum_action DISABLE TRIGGER ALL;
 918 ALTER table forum_actionrepute DISABLE TRIGGER ALL;
 919 ALTER table forum_subscriptionsettings DISABLE TRIGGER ALL;
 920 ALTER table forum_validationhash DISABLE TRIGGER ALL;
 921 ALTER table forum_authkeyuserassociation DISABLE TRIGGER ALL;
 922 ALTER table forum_tag DISABLE TRIGGER ALL;
 923 ALTER table forum_markedtag DISABLE TRIGGER ALL;
 924 ALTER table forum_node DISABLE TRIGGER ALL;
 925 ALTER table forum_nodestate DISABLE TRIGGER ALL;
 926 ALTER table forum_node_tags DISABLE TRIGGER ALL;
 927 ALTER table forum_noderevision DISABLE TRIGGER ALL;
 928 ALTER table forum_node_tags DISABLE TRIGGER ALL;
 929 ALTER table forum_questionsubscription DISABLE TRIGGER ALL;
 930 ALTER table forum_vote DISABLE TRIGGER ALL;
 931 ALTER table forum_flag DISABLE TRIGGER ALL;
 932 ALTER table forum_badge DISABLE TRIGGER ALL;
 933 ALTER table forum_award DISABLE TRIGGER ALL;
 934 ALTER table forum_openidnonce DISABLE TRIGGER ALL;
 935 ALTER table forum_openidassociation DISABLE TRIGGER ALL;
 936 """
 937
 938 PG_ENABLE_TRIGGERS = """
 939 ALTER table auth_user ENABLE TRIGGER ALL;
 940 ALTER table auth_user_groups ENABLE TRIGGER ALL;
 941 ALTER table auth_user_user_permissions ENABLE TRIGGER ALL;
 942 ALTER table forum_keyvalue ENABLE TRIGGER ALL;
 943 ALTER table forum_action ENABLE TRIGGER ALL;
 944 ALTER table forum_actionrepute ENABLE TRIGGER ALL;
 945 ALTER table forum_subscriptionsettings ENABLE TRIGGER ALL;
 946 ALTER table forum_validationhash ENABLE TRIGGER ALL;
 947 ALTER table forum_authkeyuserassociation ENABLE TRIGGER ALL;
 948 ALTER table forum_tag ENABLE TRIGGER ALL;
 949 ALTER table forum_markedtag ENABLE TRIGGER ALL;
 950 ALTER table forum_node ENABLE TRIGGER ALL;
 951 ALTER table forum_nodestate ENABLE TRIGGER ALL;
 952 ALTER table forum_node_tags ENABLE TRIGGER ALL;
 953 ALTER table forum_noderevision ENABLE TRIGGER ALL;
 954 ALTER table forum_node_tags ENABLE TRIGGER ALL;
 955 ALTER table forum_questionsubscription ENABLE TRIGGER ALL;
 956 ALTER table forum_vote ENABLE TRIGGER ALL;
 957 ALTER table forum_flag ENABLE TRIGGER ALL;
 958 ALTER table forum_badge ENABLE TRIGGER ALL;
 959 ALTER table forum_award ENABLE TRIGGER ALL;
 960 ALTER table forum_openidnonce ENABLE TRIGGER ALL;
 961 ALTER table forum_openidassociation ENABLE TRIGGER ALL;
 962 """
 963
 964 PG_SEQUENCE_RESETS = """
 965 SELECT setval('"auth_user_id_seq"', coalesce(max("id"), 1) + 2, max("id") IS NOT null) FROM "auth_user";
 966 SELECT setval('"auth_user_groups_id_seq"', coalesce(max("id"), 1) + 2, max("id") IS NOT null) FROM "auth_user_groups";
 967 SELECT setval('"auth_user_user_permissions_id_seq"', coalesce(max("id"), 1) + 2, max("id") IS NOT null) FROM "auth_user_user_permissions";
 968 SELECT setval('"forum_keyvalue_id_seq"', coalesce(max("id"), 1) + 2, max("id") IS NOT null) FROM "forum_keyvalue";
 969 SELECT setval('"forum_action_id_seq"', coalesce(max("id"), 1) + 2, max("id") IS NOT null) FROM "forum_action";
 970 SELECT setval('"forum_actionrepute_id_seq"', coalesce(max("id"), 1) + 2, max("id") IS NOT null) FROM "forum_actionrepute";
 971 SELECT setval('"forum_subscriptionsettings_id_seq"', coalesce(max("id"), 1) + 2, max("id") IS NOT null) FROM "forum_subscriptionsettings";
 972 SELECT setval('"forum_validationhash_id_seq"', coalesce(max("id"), 1) + 2, max("id") IS NOT null) FROM "forum_validationhash";
 973 SELECT setval('"forum_authkeyuserassociation_id_seq"', coalesce(max("id"), 1) + 2, max("id") IS NOT null) FROM "forum_authkeyuserassociation";
 974 SELECT setval('"forum_tag_id_seq"', coalesce(max("id"), 1) + 2, max("id") IS NOT null) FROM "forum_tag";
 975 SELECT setval('"forum_markedtag_id_seq"', coalesce(max("id"), 1) + 2, max("id") IS NOT null) FROM "forum_markedtag";
 976 SELECT setval('"forum_node_id_seq"', coalesce(max("id"), 1) + 2, max("id") IS NOT null) FROM "forum_node";
 977 SELECT setval('"forum_nodestate_id_seq"', coalesce(max("id"), 1) + 2, max("id") IS NOT null) FROM "forum_nodestate";
 978 SELECT setval('"forum_node_tags_id_seq"', coalesce(max("id"), 1) + 2, max("id") IS NOT null) FROM "forum_node_tags";
 979 SELECT setval('"forum_noderevision_id_seq"', coalesce(max("id"), 1) + 2, max("id") IS NOT null) FROM "forum_noderevision";
 980 SELECT setval('"forum_node_tags_id_seq"', coalesce(max("id"), 1) + 2, max("id") IS NOT null) FROM "forum_node_tags";
 981 SELECT setval('"forum_questionsubscription_id_seq"', coalesce(max("id"), 1) + 2, max("id") IS NOT null) FROM "forum_questionsubscription";
 982 SELECT setval('"forum_vote_id_seq"', coalesce(max("id"), 1) + 2, max("id") IS NOT null) FROM "forum_vote";
 983 SELECT setval('"forum_flag_id_seq"', coalesce(max("id"), 1) + 2, max("id") IS NOT null) FROM "forum_flag";
 984 SELECT setval('"forum_badge_id_seq"', coalesce(max("id"), 1) + 2, max("id") IS NOT null) FROM "forum_badge";
 985 SELECT setval('"forum_award_id_seq"', coalesce(max("id"), 1) + 2, max("id") IS NOT null) FROM "forum_award";
 986 SELECT setval('"forum_openidnonce_id_seq"', coalesce(max("id"), 1) + 2, max("id") IS NOT null) FROM "forum_openidnonce";
 987 SELECT setval('"forum_openidassociation_id_seq"', coalesce(max("id"), 1) + 2, max("id") IS NOT null) FROM "forum_openidassociation";
 988 """
 989
 990
 991
 992