From 23b2a2bfa95f12d0333a82c6e3d34456698522b7 Mon Sep 17 00:00:00 2001 From: hernani Date: Mon, 18 Oct 2010 12:32:59 +0000 Subject: [PATCH] Several improvements in the exporter. Finished the importer engine. Need to adapt the ui. git-svn-id: http://svn.osqa.net/svnroot/osqa/trunk@607 0cfe37f9-358a-4d5e-be75-b63607b5c754 --- forum_modules/exporter/exporter.py | 64 +-- forum_modules/exporter/importer.py | 423 ++++++++++++++++++ forum_modules/exporter/orm.py | 265 +++++++++++ .../exporter/templates/exporter.html | 4 +- .../exporter/templates/importer.html | 14 + forum_modules/exporter/urls.py | 4 +- forum_modules/exporter/views.py | 42 +- 7 files changed, 770 insertions(+), 46 deletions(-) create mode 100644 forum_modules/exporter/importer.py create mode 100644 forum_modules/exporter/orm.py create mode 100644 forum_modules/exporter/templates/importer.html diff --git a/forum_modules/exporter/exporter.py b/forum_modules/exporter/exporter.py index 84905b3..5559e44 100644 --- a/forum_modules/exporter/exporter.py +++ b/forum_modules/exporter/exporter.py @@ -21,7 +21,8 @@ LAST_BACKUP = os.path.join(TMP_FOLDER, 'backup.tar.gz') DATE_AND_AUTHOR_INF_SECTION = 'DateAndAuthor' OPTIONS_INF_SECTION = 'Options' -DATETIME_FORMAT = "%a %b %d %H:%M:%S %Y" +DATETIME_FORMAT = "%Y-%m-%d %H:%M:%S" +DATE_FORMAT = "%Y-%m-%d" def Etree_pretty__write(self, file, node, encoding, namespaces, level=0, identator=" "): @@ -87,7 +88,7 @@ def ET_Element_add_tag(el, tag_name, content = None, **attrs): tag = ET.SubElement(el, tag_name) if content: - tag.text = unicode(content) + tag.text = unicode(content).encode('utf-8') for k, v in attrs.items(): tag.set(k, unicode(v)) @@ -101,7 +102,7 @@ def make_extra(el, v): return - if isinstance(v, (int, long, str, float, bool, dict, list, tuple)): + if isinstance(v, (int, long, str, unicode, float, bool, dict, list, tuple)): if isinstance(v, tuple): t = 'list' else: @@ -159,13 +160,13 @@ def create_targz(tmp, files, start_time, options, user, state, set_state): else: domain = 'localhost' - fname = "%s-%s.tar.gz" % (domain, now.strftime('%Y%m%d%H%M')) + fname = "%s-%s" % (domain, now.strftime('%Y%m%d%H%M')) inf = ConfigParser.SafeConfigParser() inf.add_section(DATE_AND_AUTHOR_INF_SECTION) - inf.set(DATE_AND_AUTHOR_INF_SECTION, 'file-name', fname) + inf.set(DATE_AND_AUTHOR_INF_SECTION, 'file-name', "%s.tar.gz" % fname) inf.set(DATE_AND_AUTHOR_INF_SECTION, 'author', unicode(user.id)) inf.set(DATE_AND_AUTHOR_INF_SECTION, 'site', djsettings.APP_URL) inf.set(DATE_AND_AUTHOR_INF_SECTION, 'started', start_time.strftime(DATETIME_FORMAT)) @@ -183,7 +184,9 @@ def create_targz(tmp, files, start_time, options, user, state, set_state): state['overall']['status'] = _('Saving backup file') set_state() t.close() - shutil.copyfile(LAST_BACKUP, os.path.join(selfsettings.EXPORTER_BACKUP_STORAGE, fname)) + shutil.copyfile(LAST_BACKUP, os.path.join(selfsettings.EXPORTER_BACKUP_STORAGE, "%s.tar.gz" % fname)) + shutil.copyfile(os.path.join(tmp, 'backup.inf'), os.path.join(selfsettings.EXPORTER_BACKUP_STORAGE, "%s.backup.inf" % fname)) + def export_upfiles(tf): @@ -223,7 +226,7 @@ def export(options, user): def set_state(): full_state['time_started'] = diff_date(start_time) - cache.set(CACHE_KEY, full_state, 60) + cache.set(CACHE_KEY, full_state) set_state() @@ -275,7 +278,6 @@ def export(options, user): import traceback logging.error("Error executing xml backup: \n %s" % (traceback.format_exc())) - print traceback.format_exc() finally: xml.etree.ElementTree.ElementTree._write = original__write del xml.etree.ElementTree._ElementInterface.add @@ -329,14 +331,15 @@ def export_users(u, el, anon_data): el.add('password', u.password) el.add('email', u.email, validated=u.email_isvalid and 'true' or 'false') el.add('reputation', u.reputation) - el.add('joindate', u.date_joined) + el.add('badges', bronze=u.bronze, silver=u.silver, gold=u.gold) + el.add('joindate', u.date_joined.strftime(DATETIME_FORMAT)) + el.add('active', u.is_active and 'true' or 'false') - el.add('firstname', u.first_name) - el.add('lastname', u.last_name) + el.add('realname', u.real_name) el.add('bio', u.about) el.add('location', u.location) el.add('website', u.website) - el.add('birthdate', u.date_of_birth) + el.add('birthdate', u.date_of_birth and u.date_of_birth.strftime(DATE_FORMAT) or "") roles = el.add('roles') @@ -385,8 +388,13 @@ def export_nodes(n, el, anon_data): if not anon_data: el.add('author', n.author.id) - el.add('date', n.added_at) + el.add('date', n.added_at.strftime(DATETIME_FORMAT)) el.add('parent', n.parent and n.parent.id or "") + el.add('absparent', n.abs_parent and n.abs_parent or "") + + act = el.add('lastactivity') + act.add('by', n.last_activity_by and n.last_activity_by.id or "") + act.add('at', n.last_activity_at and n.last_activity_at.strftime(DATETIME_FORMAT) or "") el.add('title', n.title) el.add('body', n.body) @@ -396,7 +404,7 @@ def export_nodes(n, el, anon_data): for t in n.tagname_list(): tags.add('tag', t) - revs = el.add('revisions', active=n.active_revision and n.active_revision or n.revisions.order_by('revision')[0]) + revs = el.add('revisions', active=n.active_revision and n.active_revision.revision or n.revisions.order_by('revision')[0].revision) for r in n.revisions.order_by('revision'): rev = _add_tag(revs, 'revision') @@ -404,14 +412,16 @@ def export_nodes(n, el, anon_data): rev.add('summary', r.summary) if not anon_data: rev.add('author', r.author.id) - rev.add('date', r.revised_at) + rev.add('date', r.revised_at.strftime(DATETIME_FORMAT)) rev.add('title', r.title) rev.add('body', r.body) rev.add('tags', ", ".join(r.tagname_list())) + el.add('marked', n.marked and 'true' or 'false') el.add('extraRef', n.extra_ref and n.extra_ref.id or "") - make_extra(el.add('exraData'), n.extra) + make_extra(el.add('extraData'), n.extra) + el.add('extraCount', n.extra_count and n.extra_count or "") @exporter_step(Action.objects.all(), 'actions', 'action', _('Actions'), 'action_date') @@ -446,24 +456,24 @@ def export_actions(a, el, anon_data): repute.add('value', r.value) -@exporter_step(NodeState.objects.all(), 'states', 'state', _('Node states'), 'action__action_date') -def export_states(s, el, anon_data): - el.add('type', s.state_type) - el.add('node', s.node.id) - el.add('trigger', s.action.id) +#@exporter_step(NodeState.objects.all(), 'states', 'state', _('Node states'), 'action__action_date') +#def export_states(s, el, anon_data): +# el.add('type', s.state_type) +# el.add('node', s.node.id) +# el.add('trigger', s.action.id) -@exporter_step(Badge.objects.all(), 'badges', 'badge', _('Badges'), user_data=True) -def export_badges(b, el, anon_data): - el.add('type', ["", 'gold', 'silver', 'bronze'][b.type]) - el.add('name', b.cls) - el.add('count', b.awarded_count) +#@exporter_step(Badge.objects.all(), 'badges', 'badge', _('Badges'), user_data=True) +#def export_badges(b, el, anon_data): +# el.add('type', ["", 'gold', 'silver', 'bronze'][b.type]) +# el.add('name', b.cls) +# el.add('count', b.awarded_count) @exporter_step(Award.objects.all(), 'awards', 'award', _('Awards'), 'awarded_at', True) def export_awards(a, el, anon_data): el.add('badge', a.badge.cls) - el.add('user', a.user) + el.add('user', a.user.id) el.add('node', a.node and a.node.id or "") el.add('trigger', a.trigger and a.trigger.id or "") el.add('action', a.action.id) diff --git a/forum_modules/exporter/importer.py b/forum_modules/exporter/importer.py new file mode 100644 index 0000000..35b42b1 --- /dev/null +++ b/forum_modules/exporter/importer.py @@ -0,0 +1,423 @@ +import os, tarfile, datetime + +from xml.sax import make_parser +from xml.sax.handler import ContentHandler, ErrorHandler + +from exporter import TMP_FOLDER, DATETIME_FORMAT, DATE_FORMAT +from orm import orm + +NO_DEFAULT = object() + +class ContentElement(): + def __init__(self, content): + self._content = content + + def content(self): + return self._content.strip() + + def as_bool(self): + return self.content() == "true" + + def as_date(self, default=NO_DEFAULT): + try: + return datetime.datetime.strptime(self.content(), DATE_FORMAT) + except: + if default == NO_DEFAULT: + return datetime.date.fromtimestamp(0) + else: + return default + + + def as_datetime(self, default=NO_DEFAULT): + try: + return datetime.datetime.strptime(self.content(), DATETIME_FORMAT) + except: + if default == NO_DEFAULT: + return datetime.datetime.fromtimestamp(0) + else: + return default + + def as_int(self, default=0): + try: + return int(self.content()) + except: + return default + + def __str__(self): + return self.content() + + +class RowElement(ContentElement): + def __init__(self, name, attrs, parent=None): + self.name = name.lower() + self.parent = parent + self.attrs = dict([(k.lower(), ContentElement(v)) for k, v in attrs.items()]) + self._content = '' + self.sub_elements = {} + + if parent: + parent.add(self) + + def add_to_content(self, ch): + self._content += ch + + def add(self, sub): + curr = self.sub_elements.get(sub.name, None) + + if not curr: + curr = [] + self.sub_elements[sub.name] = curr + + curr.append(sub) + + def get(self, name, default=None): + return self.sub_elements.get(name.lower(), [default])[-1] + + def get_list(self, name): + return self.sub_elements.get(name.lower(), []) + + def get_listc(self, name): + return [r.content() for r in self.get_list(name)] + + def getc(self, name, default=""): + el = self.get(name, None) + + if el: + return el.content() + else: + return default + + def get_attr(self, name, default=""): + return self.attrs.get(name.lower(), default) + + def as_pickled(self, default=None): + value_el = self.get('value') + + if value_el: + return value_el._as_pickled(default) + else: + return default + + TYPES_MAP = dict([(c.__name__, c) for c in (int, long, str, unicode, float)]) + + def _as_pickled(self, default=None): + type = self.get_attr('type').content() + + try: + if type == 'dict': + return dict([ (item.get_attr('key'), item.as_pickled()) for item in self.get_list('item') ]) + elif type == 'list': + return [item.as_pickled() for item in self.get_list('item')] + elif type == 'bool': + return self.content().lower() == 'true' + elif type in RowElement.TYPES_MAP: + return RowElement.TYPES_MAP[type](self.content()) + else: + return self.content() + except: + return default + + + + +class TableHandler(ContentHandler): + def __init__(self, root_name, row_name, callback, callback_args = []): + self.root_name = root_name.lower() + self.row_name = row_name.lower() + self.callback = callback + self.callback_args = callback_args + + self._reset() + + def _reset(self): + self.curr_element = None + self.in_tag = None + + def startElement(self, name, attrs): + name = name.lower() + + if name == self.root_name.lower(): + pass + elif name == self.row_name: + self.curr_element = RowElement(name, attrs) + else: + self.curr_element = RowElement(name, attrs, self.curr_element) + + def characters(self, ch): + if self.curr_element: + self.curr_element.add_to_content(ch) + + def endElement(self, name): + name = name.lower() + + if name == self.root_name: + pass + elif name == self.row_name: + self.callback(self.curr_element, *self.callback_args) + self._reset() + else: + self.curr_element = self.curr_element.parent + + +class SaxErrorHandler(ErrorHandler): + def error(self, e): + raise e + + def fatalError(self, e): + raise e + + def warning(self, e): + raise e + +FILE_HANDLERS = [] + +def start_import(fname, user): + #dump = tarfile.open(fname, 'r') + #dump.extractall(TMP_FOLDER) + + for h in FILE_HANDLERS: + h(TMP_FOLDER, user) + +def file_handler(file_name, root_tag, el_tag, args_handler=None, pre_callback=None, post_callback=None): + def decorator(fn): + def decorated(location, current_user): + if pre_callback: + pre_callback(current_user) + + if (args_handler): + args = args_handler(current_user) + else: + args = [] + + parser = make_parser() + handler = TableHandler(root_tag, el_tag, fn, args) + parser.setContentHandler(handler) + #parser.setErrorHandler(SaxErrorHandler()) + + parser.parse(os.path.join(location, file_name)) + + if post_callback: + post_callback() + + FILE_HANDLERS.append(decorated) + return decorated + return decorator + + +@file_handler('users.xml', 'users', 'user', args_handler=lambda u: [u]) +def user_import(row, current_user): + if str(current_user.id) == row.getc('id'): + return + + roles = row.get('roles').get_listc('role') + valid_email = row.get('email').get_attr('validated').as_bool() + badges = row.get('badges') + + user = orm.User( + id = row.getc('id'), + username = row.getc('username'), + password = row.getc('password'), + email = row.getc('email'), + email_isvalid= valid_email, + is_superuser = 'superuser' in roles, + is_staff = 'moderator' in roles, + is_active = True, + date_joined = row.get('joindate').as_datetime(), + about = row.getc('bio'), + date_of_birth = row.get('birthdate').as_date(None), + website = row.getc('website'), + reputation = row.get('reputation').as_int(), + gold = badges.get_attr('gold').as_int(), + silver = badges.get_attr('silver').as_int(), + bronze = badges.get_attr('bronze').as_int(), + real_name = row.getc('realname'), + location = row.getc('location'), + ) + + user.save() + + authKeys = row.get('authKeys') + + for key in authKeys.get_list('key'): + orm.AuthKeyUserAssociation(user=user, key=key.getc('key'), provider=key.getc('provider')).save() + + notifications = row.get('notifications') + + attributes = dict([(str(k), v.as_bool() and 'i' or 'n') for k, v in notifications.get('notify').attrs.items()]) + attributes.update(dict([(str(k), v.as_bool()) for k, v in notifications.get('autoSubscribe').attrs.items()])) + attributes.update(dict([(str("notify_%s" % k), v.as_bool()) for k, v in notifications.get('notifyOnSubscribed').attrs.items()])) + + orm.SubscriptionSettings(user=user, enable_notifications=notifications.get_attr('enabled').as_bool(), **attributes).save() + +def pre_tag_import(user): + tag_import.tag_mappings={} + + +@file_handler('tags.xml', 'tags', 'tag', pre_callback=pre_tag_import) +def tag_import(row): + tag = orm.Tag(name=row.getc('name'), used_count=row.get('used').as_int(), created_by_id=row.get('author').as_int()) + tag.save() + tag_import.tag_mappings[tag.name] = tag + + +def post_node_import(): + tag_import.tag_mappings = None + +@file_handler('nodes.xml', 'nodes', 'node', args_handler=lambda u: [tag_import.tag_mappings], post_callback=post_node_import) +def node_import(row, tags): + + ntags = [] + + for t in row.get('tags').get_list('tag'): + ntags.append(tags[t.content()]) + + last_act = row.get('lastactivity') + + node = orm.Node( + id = row.getc('id'), + node_type = row.getc('type'), + author_id = row.get('author').as_int(), + added_at = row.get('date').as_datetime(), + parent_id = row.get('parent').as_int(None), + abs_parent_id = row.get('absparent').as_int(None), + + last_activity_by_id = last_act.get('by').as_int(None), + last_activity_at = last_act.get('at').as_datetime(None), + + title = row.getc('title'), + body = row.getc('body'), + tagnames = " ".join([t.name for t in ntags]), + + marked = row.get('marked').as_bool(), + extra_ref_id = row.get('extraRef').as_int(None), + extra_count = row.get('extraCount').as_int(0), + extra = row.get('extraData').as_pickled() + ) + + node.save() + node.tags = ntags + + revisions = row.get('revisions') + active = revisions.get_attr('active').as_int() + + for r in revisions.get_list('revision'): + rev = orm.NodeRevision( + author_id = r.getc('author'), + body = r.getc('body'), + node = node, + revised_at = r.get('date').as_datetime(), + revision = r.get('number').as_int(), + summary = r.getc('summary'), + tagnames = " ".join(r.getc('tags').split(',')), + title = r.getc('title'), + ) + + rev.save() + if rev.revision == active: + active = rev + + node.active_revision = active + node.save() + +POST_ACTION = {} + +def post_action(*types): + def decorator(fn): + for t in types: + POST_ACTION[t] = fn + return fn + return decorator + +def post_action_import_callback(): + with_state = orm.Node.objects.filter(id__in=orm.NodeState.objects.values_list('node_id', flat=True).distinct()) + + for n in with_state: + n.state_string = "".join(["(%s)" % s for s in n.states.values_list('state_type')]) + n.save() + +@file_handler('actions.xml', 'actions', 'action', post_callback=post_action_import_callback) +def actions_import(row): + action = orm.Action( + id = row.get('id').as_int(), + action_type = row.getc('type'), + action_date = row.get('date').as_datetime(), + node_id = row.get('node').as_int(None), + user_id = row.get('user').as_int(), + real_user_id = row.get('realUser').as_int(None), + ip = row.getc('ip'), + extra = row.get('extraData').as_pickled(), + ) + + canceled = row.get('canceled') + if canceled.get_attr('state').as_bool(): + action.canceled_by_id = canceled.get('user').as_int() + action.canceled_at = canceled.get('date').as_datetime(), + action.canceled_ip = canceled.getc('ip') + + action.save() + + for r in row.get('reputes').get_list('repute'): + by_canceled = r.get_attr('byCanceled').as_bool() + + orm.ActionRepute( + action = action, + user_id = r.get('user').as_int(), + value = r.get('value').as_int(), + + date = by_canceled and action.canceled_at or action.action_date, + by_canceled = by_canceled + ).save() + + if (not action.canceled) and action.action_type in POST_ACTION: + POST_ACTION[action.action_type](row, action) + + + + +@post_action('voteup', 'votedown', 'voteupcomment') +def vote_action(row, action): + orm.Vote(user_id=action.user_id, node_id=action.node_id, action=action, + voted_at=action.action_date, value=(action.action_type != 'votedown') and 1 or -1).save() + +def state_action(state): + def fn(row, action): + orm.NodeState( + state_type = state, + node_id = action.node_id, + action = action + ).save() + return fn + +post_action('wikify')(state_action('wiki')) +post_action('delete')(state_action('deleted')) +post_action('acceptanswer')(state_action('accepted')) +post_action('publish')(state_action('published')) + + +@post_action('flag') +def flag_action(row, action): + orm.Flag(user_id=action.user_id, node_id=action.node_id, action=action, reason=action.extra).save() + + +def award_import_args(user): + return [ dict([ (b.cls, b) for b in orm.Badge.objects.all() ]) ] + + +@file_handler('awards.xml', 'awards', 'award', args_handler=award_import_args) +def awards_import(row, badges): + award = orm.Award( + user_id = row.get('user').as_int(), + badge = badges[row.getc('badge')], + node_id = row.get('node').as_int(None), + action_id = row.get('action').as_int(None), + trigger_id = row.get('trigger').as_int(None) + ).save() + + + + + + + + + diff --git a/forum_modules/exporter/orm.py b/forum_modules/exporter/orm.py new file mode 100644 index 0000000..ff23c85 --- /dev/null +++ b/forum_modules/exporter/orm.py @@ -0,0 +1,265 @@ +from south.v2 import DataMigration +from south.orm import FakeORM + +class Migration(DataMigration): + def forwards(self, orm): + pass + + + def backwards(self, orm): + "Write your backwards methods here." + + models = { + 'auth.group': { + 'Meta': {'object_name': 'Group'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}), + 'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}) + }, + 'auth.permission': { + 'Meta': {'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'}, + 'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '50'}) + }, + 'auth.user': { + 'Meta': {'object_name': 'User'}, + 'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}), + 'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True', 'blank': 'True'}), + 'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'blank': 'True'}), + 'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'blank': 'True'}), + 'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}), + 'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'}) + }, + 'contenttypes.contenttype': { + 'Meta': {'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"}, + 'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}) + }, + 'forum.action': { + 'Meta': {'object_name': 'Action'}, + 'action_date': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'action_type': ('django.db.models.fields.CharField', [], {'max_length': '16'}), + 'canceled': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'blank': 'True'}), + 'canceled_at': ('django.db.models.fields.DateTimeField', [], {'null': 'True'}), + 'canceled_by': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'canceled_actions'", 'null': 'True', 'to': "orm['forum.User']"}), + 'canceled_ip': ('django.db.models.fields.CharField', [], {'max_length': '16'}), + 'extra': ('forum.models.utils.PickledObjectField', [], {'null': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'ip': ('django.db.models.fields.CharField', [], {'max_length': '16'}), + 'node': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'actions'", 'null': 'True', 'to': "orm['forum.Node']"}), + 'real_user': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'proxied_actions'", 'null': 'True', 'to': "orm['forum.User']"}), + 'user': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'actions'", 'to': "orm['forum.User']"}) + }, + 'forum.actionrepute': { + 'Meta': {'object_name': 'ActionRepute'}, + 'action': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'reputes'", 'to': "orm['forum.Action']"}), + 'by_canceled': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'blank': 'True'}), + 'date': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'user': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'reputes'", 'to': "orm['forum.User']"}), + 'value': ('django.db.models.fields.IntegerField', [], {'default': '0'}) + }, + 'forum.authkeyuserassociation': { + 'Meta': {'object_name': 'AuthKeyUserAssociation'}, + 'added_at': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'key': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}), + 'provider': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'user': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'auth_keys'", 'to': "orm['forum.User']"}) + }, + 'forum.award': { + 'Meta': {'unique_together': "(('user', 'badge', 'node'),)", 'object_name': 'Award'}, + 'action': ('django.db.models.fields.related.OneToOneField', [], {'related_name': "'award'", 'unique': 'True', 'to': "orm['forum.Action']"}), + 'awarded_at': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'badge': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'awards'", 'to': "orm['forum.Badge']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'node': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['forum.Node']", 'null': 'True'}), + 'trigger': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'awards'", 'null': 'True', 'to': "orm['forum.Action']"}), + 'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['forum.User']"}) + }, + 'forum.badge': { + 'Meta': {'object_name': 'Badge'}, + 'awarded_count': ('django.db.models.fields.PositiveIntegerField', [], {'default': '0'}), + 'awarded_to': ('django.db.models.fields.related.ManyToManyField', [], {'related_name': "'badges'", 'symmetrical': 'False', 'through': "orm['forum.Award']", 'to': "orm['forum.User']"}), + 'cls': ('django.db.models.fields.CharField', [], {'max_length': '50', 'null': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'type': ('django.db.models.fields.SmallIntegerField', [], {}) + }, + 'forum.flag': { + 'Meta': {'unique_together': "(('user', 'node'),)", 'object_name': 'Flag'}, + 'action': ('django.db.models.fields.related.OneToOneField', [], {'related_name': "'flag'", 'unique': 'True', 'to': "orm['forum.Action']"}), + 'flagged_at': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'node': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'flags'", 'to': "orm['forum.Node']"}), + 'reason': ('django.db.models.fields.CharField', [], {'max_length': '300'}), + 'user': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'flags'", 'to': "orm['forum.User']"}) + }, + 'forum.keyvalue': { + 'Meta': {'object_name': 'KeyValue'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'key': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}), + 'value': ('forum.models.utils.PickledObjectField', [], {'null': 'True'}) + }, + 'forum.markedtag': { + 'Meta': {'object_name': 'MarkedTag'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'reason': ('django.db.models.fields.CharField', [], {'max_length': '16'}), + 'tag': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'user_selections'", 'to': "orm['forum.Tag']"}), + 'user': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'tag_selections'", 'to': "orm['forum.User']"}) + }, + 'forum.node': { + 'Meta': {'object_name': 'Node'}, + 'abs_parent': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'all_children'", 'null': 'True', 'to': "orm['forum.Node']"}), + 'active_revision': ('django.db.models.fields.related.OneToOneField', [], {'related_name': "'active'", 'unique': 'True', 'null': 'True', 'to': "orm['forum.NodeRevision']"}), + 'added_at': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'author': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'nodes'", 'to': "orm['forum.User']"}), + 'body': ('django.db.models.fields.TextField', [], {}), + 'extra': ('forum.models.utils.PickledObjectField', [], {'null': 'True'}), + 'extra_count': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'extra_ref': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['forum.Node']", 'null': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'last_activity_at': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}), + 'last_activity_by': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['forum.User']", 'null': 'True'}), + 'last_edited': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'edited_node'", 'unique': 'True', 'null': 'True', 'to': "orm['forum.Action']"}), + 'marked': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'blank': 'True'}), + 'node_type': ('django.db.models.fields.CharField', [], {'default': "'node'", 'max_length': '16'}), + 'parent': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'children'", 'null': 'True', 'to': "orm['forum.Node']"}), + 'score': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'state_string': ('django.db.models.fields.TextField', [], {'default': "''"}), + 'tagnames': ('django.db.models.fields.CharField', [], {'max_length': '125'}), + 'tags': ('django.db.models.fields.related.ManyToManyField', [], {'related_name': "'nodes'", 'symmetrical': 'False', 'to': "orm['forum.Tag']"}), + 'title': ('django.db.models.fields.CharField', [], {'max_length': '300'}) + }, + 'forum.noderevision': { + 'Meta': {'unique_together': "(('node', 'revision'),)", 'object_name': 'NodeRevision'}, + 'author': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'noderevisions'", 'to': "orm['forum.User']"}), + 'body': ('django.db.models.fields.TextField', [], {}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'node': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'revisions'", 'to': "orm['forum.Node']"}), + 'revised_at': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'revision': ('django.db.models.fields.PositiveIntegerField', [], {}), + 'summary': ('django.db.models.fields.CharField', [], {'max_length': '300'}), + 'tagnames': ('django.db.models.fields.CharField', [], {'max_length': '125'}), + 'title': ('django.db.models.fields.CharField', [], {'max_length': '300'}) + }, + 'forum.nodestate': { + 'Meta': {'unique_together': "(('node', 'state_type'),)", 'object_name': 'NodeState'}, + 'action': ('django.db.models.fields.related.OneToOneField', [], {'related_name': "'node_state'", 'unique': 'True', 'to': "orm['forum.Action']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'node': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'states'", 'to': "orm['forum.Node']"}), + 'state_type': ('django.db.models.fields.CharField', [], {'max_length': '16'}) + }, + 'forum.openidassociation': { + 'Meta': {'object_name': 'OpenIdAssociation'}, + 'assoc_type': ('django.db.models.fields.TextField', [], {'max_length': '64'}), + 'handle': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'issued': ('django.db.models.fields.IntegerField', [], {}), + 'lifetime': ('django.db.models.fields.IntegerField', [], {}), + 'secret': ('django.db.models.fields.TextField', [], {'max_length': '255'}), + 'server_url': ('django.db.models.fields.TextField', [], {'max_length': '2047'}) + }, + 'forum.openidnonce': { + 'Meta': {'object_name': 'OpenIdNonce'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'salt': ('django.db.models.fields.CharField', [], {'max_length': '50'}), + 'server_url': ('django.db.models.fields.URLField', [], {'max_length': '200'}), + 'timestamp': ('django.db.models.fields.IntegerField', [], {}) + }, + 'forum.questionsubscription': { + 'Meta': {'object_name': 'QuestionSubscription'}, + 'auto_subscription': ('django.db.models.fields.BooleanField', [], {'default': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'last_view': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime(2010, 7, 1, 13, 6, 46, 789996)'}), + 'question': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['forum.Node']"}), + 'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['forum.User']"}) + }, + 'forum.subscriptionsettings': { + 'Meta': {'object_name': 'SubscriptionSettings'}, + 'all_questions': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'blank': 'True'}), + 'all_questions_watched_tags': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'blank': 'True'}), + 'enable_notifications': ('django.db.models.fields.BooleanField', [], {'default': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'member_joins': ('django.db.models.fields.CharField', [], {'default': "'n'", 'max_length': '1'}), + 'new_question': ('django.db.models.fields.CharField', [], {'default': "'d'", 'max_length': '1'}), + 'new_question_watched_tags': ('django.db.models.fields.CharField', [], {'default': "'i'", 'max_length': '1'}), + 'notify_accepted': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'blank': 'True'}), + 'notify_answers': ('django.db.models.fields.BooleanField', [], {'default': 'True', 'blank': 'True'}), + 'notify_comments': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'blank': 'True'}), + 'notify_comments_own_post': ('django.db.models.fields.BooleanField', [], {'default': 'True', 'blank': 'True'}), + 'notify_reply_to_comments': ('django.db.models.fields.BooleanField', [], {'default': 'True', 'blank': 'True'}), + 'questions_answered': ('django.db.models.fields.BooleanField', [], {'default': 'True', 'blank': 'True'}), + 'questions_asked': ('django.db.models.fields.BooleanField', [], {'default': 'True', 'blank': 'True'}), + 'questions_commented': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'blank': 'True'}), + 'questions_viewed': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'blank': 'True'}), + 'send_digest': ('django.db.models.fields.BooleanField', [], {'default': 'True', 'blank': 'True'}), + 'subscribed_questions': ('django.db.models.fields.CharField', [], {'default': "'i'", 'max_length': '1'}), + 'user': ('django.db.models.fields.related.OneToOneField', [], {'related_name': "'subscription_settings'", 'unique': 'True', 'to': "orm['forum.User']"}) + }, + 'forum.tag': { + 'Meta': {'object_name': 'Tag'}, + 'created_by': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'created_tags'", 'to': "orm['forum.User']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'marked_by': ('django.db.models.fields.related.ManyToManyField', [], {'related_name': "'marked_tags'", 'symmetrical': 'False', 'through': "orm['forum.MarkedTag']", 'to': "orm['forum.User']"}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}), + 'used_count': ('django.db.models.fields.PositiveIntegerField', [], {'default': '0'}) + }, + 'forum.user': { + 'Meta': {'object_name': 'User', '_ormbases': ['auth.User']}, + 'about': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'bronze': ('django.db.models.fields.PositiveIntegerField', [], {'default': '0'}), + 'date_of_birth': ('django.db.models.fields.DateField', [], {'null': 'True', 'blank': 'True'}), + 'email_isvalid': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'blank': 'True'}), + 'gold': ('django.db.models.fields.PositiveIntegerField', [], {'default': '0'}), + 'is_approved': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'blank': 'True'}), + 'last_seen': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'location': ('django.db.models.fields.CharField', [], {'max_length': '100', 'blank': 'True'}), + 'real_name': ('django.db.models.fields.CharField', [], {'max_length': '100', 'blank': 'True'}), + 'reputation': ('django.db.models.fields.PositiveIntegerField', [], {'default': '0'}), + 'silver': ('django.db.models.fields.PositiveIntegerField', [], {'default': '0'}), + 'subscriptions': ('django.db.models.fields.related.ManyToManyField', [], {'related_name': "'subscribers'", 'symmetrical': 'False', 'through': "orm['forum.QuestionSubscription']", 'to': "orm['forum.Node']"}), + 'user_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['auth.User']", 'unique': 'True', 'primary_key': 'True'}), + 'website': ('django.db.models.fields.URLField', [], {'max_length': '200', 'blank': 'True'}) + }, + 'forum.userproperty': { + 'Meta': {'unique_together': "(('user', 'key'),)", 'object_name': 'UserProperty'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'key': ('django.db.models.fields.CharField', [], {'max_length': '16'}), + 'user': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'properties'", 'to': "orm['forum.User']"}), + 'value': ('forum.models.utils.PickledObjectField', [], {'null': 'True'}) + }, + 'forum.validationhash': { + 'Meta': {'unique_together': "(('user', 'type'),)", 'object_name': 'ValidationHash'}, + 'expiration': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime(2010, 7, 2, 13, 6, 46, 883626)'}), + 'hash_code': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'seed': ('django.db.models.fields.CharField', [], {'max_length': '12'}), + 'type': ('django.db.models.fields.CharField', [], {'max_length': '12'}), + 'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['forum.User']"}) + }, + 'forum.vote': { + 'Meta': {'unique_together': "(('user', 'node'),)", 'object_name': 'Vote'}, + 'action': ('django.db.models.fields.related.OneToOneField', [], {'related_name': "'vote'", 'unique': 'True', 'to': "orm['forum.Action']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'node': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'votes'", 'to': "orm['forum.Node']"}), + 'user': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'votes'", 'to': "orm['forum.User']"}), + 'value': ('django.db.models.fields.SmallIntegerField', [], {}), + 'voted_at': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}) + } + } + + complete_apps = ['forum'] + +orm = FakeORM(Migration, "forum") + diff --git a/forum_modules/exporter/templates/exporter.html b/forum_modules/exporter/templates/exporter.html index c7244cb..78c12c7 100644 --- a/forum_modules/exporter/templates/exporter.html +++ b/forum_modules/exporter/templates/exporter.html @@ -10,7 +10,7 @@ {% endblock %} {% block admincontent %} -{% comment %}

+

{% trans "Available backups" %}

{% endcomment %} +

{% trans "Start new backup" %} diff --git a/forum_modules/exporter/templates/importer.html b/forum_modules/exporter/templates/importer.html new file mode 100644 index 0000000..d3fd942 --- /dev/null +++ b/forum_modules/exporter/templates/importer.html @@ -0,0 +1,14 @@ +{% extends basetemplate %} + +{% load i18n %} + +{% block subtitle %} + {% trans "XML data importer" %} +{% endblock %} +{% block description %} + {% trans "Import data from dump file" %} +{% endblock %} + +{% block admincontent %} + +{% endblock %} \ No newline at end of file diff --git a/forum_modules/exporter/urls.py b/forum_modules/exporter/urls.py index 1ae0cd8..9029aec 100644 --- a/forum_modules/exporter/urls.py +++ b/forum_modules/exporter/urls.py @@ -2,10 +2,12 @@ from django.conf.urls.defaults import * from django.views.generic.simple import direct_to_template from django.utils.translation import ugettext as _ -from views import state, running, download +from views import state, running, download, importer urlpatterns = patterns('', url(r'^%s%s%s$' % (_('admin/'), _('exporter/'), _('state/')), state, name='exporter_state'), url(r'^%s%s%s$' % (_('admin/'), _('exporter/'), _('running/')), running, name='exporter_running'), url(r'^%s%s%s$' % (_('admin/'), _('exporter/'), _('download/')), download, name='exporter_download'), + + url(r'^%s%s%s$' % (_('admin/'), _('exporter/'), _('import/')), importer, name='exporter_import'), ) \ No newline at end of file diff --git a/forum_modules/exporter/views.py b/forum_modules/exporter/views.py index fe8086f..947d8ef 100644 --- a/forum_modules/exporter/views.py +++ b/forum_modules/exporter/views.py @@ -14,6 +14,7 @@ import settings as selsettings from forum import settings from exporter import export, CACHE_KEY, EXPORT_STEPS, LAST_BACKUP, DATE_AND_AUTHOR_INF_SECTION, DATETIME_FORMAT +from importer import start_import @admin_tools_page(_('exporter'), _('XML data export')) @@ -37,22 +38,23 @@ def exporter(request): available = [] - #folder = unicode(selsettings.EXPORTER_BACKUP_STORAGE) - - #for f in os.listdir(folder): - # if (not os.path.isdir(os.path.join(folder, f))) and f.endswith('.tar.gz'): - # try: - # tar = tarfile.open(os.path.join(folder, f), "r") - # inf = ConfigParser.SafeConfigParser() - # inf.readfp(tar.extractfile('backup.inf')) - # - # if inf.get(DATE_AND_AUTHOR_INF_SECTION, 'site') == settings.APP_URL: - # available.append({ - # 'author': User.objects.get(id=inf.get(DATE_AND_AUTHOR_INF_SECTION, 'author')), - # 'date': datetime.strptime(inf.get(DATE_AND_AUTHOR_INF_SECTION, 'finished'), ) - # }) - # except Exception, e: - # pass + folder = unicode(selsettings.EXPORTER_BACKUP_STORAGE) + + for f in os.listdir(folder): + if (not os.path.isdir(os.path.join(folder, f))) and f.endswith('.backup.inf'): + try: + with open(os.path.join(folder, f), 'r') as inffile: + inf = ConfigParser.SafeConfigParser() + inf.readfp(inffile) + + if inf.get(DATE_AND_AUTHOR_INF_SECTION, 'site') == settings.APP_URL and os.path.exists( + os.path.join(folder, inf.get(DATE_AND_AUTHOR_INF_SECTION, 'file-name'))): + available.append({ + 'author': User.objects.get(id=inf.get(DATE_AND_AUTHOR_INF_SECTION, 'author')), + 'date': datetime.datetime.strptime(inf.get(DATE_AND_AUTHOR_INF_SECTION, 'finished'), DATETIME_FORMAT) + }) + except Exception, e: + pass return ('modules/exporter/exporter.html', { 'form': form, @@ -85,4 +87,12 @@ def download(request): return response +@admin_page +def importer(request): + start_import('/Users/admin/dev/pyenv/osqa/maintain/forum_modules/exporter/backups/localhost-201010121118.tar.gz', request.user) + + return ('modules/exporter/importer.html', { + + }) + -- 2.45.1