Ronnie has proposed merging lp:~ronnie.vd.c/loco-directory/611304 into
lp:loco-directory.
Requested reviews:
loco-directory-dev (loco-directory-dev)
Related bugs:
#611304 Import data from loco.ubuntu.com JSON service locally
https://bugs.launchpad.net/bugs/611304
Importing data from the Live (loco.ubuntu.com) server and put it in the local
database
--
https://code.launchpad.net/~ronnie.vd.c/loco-directory/611304/+merge/44267
Your team loco-directory-dev is requested to review the proposed merge of
lp:~ronnie.vd.c/loco-directory/611304 into lp:loco-directory.
=== added file 'loco_directory/teams/management/commands/import-live-data.py'
--- loco_directory/teams/management/commands/import-live-data.py 1970-01-01 00:00:00 +0000
+++ loco_directory/teams/management/commands/import-live-data.py 2010-12-20 19:19:53 +0000
@@ -0,0 +1,109 @@
+#!/usr/bin/python
+
+from django.core.management.base import NoArgsCommand
+from django.contrib.auth.models import User, Group
+from events.models import GlobalEvent, TeamEvent, TeamEventComment, Attendee
+from teams.models import Language, Continent, Country, Team
+from venues.models import Venue
+from userprofiles.models import UserProfile
+
+import sys
+import urllib2
+import json
+import datetime
+import re
+
+ORDER = [('languages', Language),
+ ('continents', Continent),
+ ('countries', Country),
+ ('venues', Venue),
+ ('global', GlobalEvent),
+ ('groups', Group),
+ ('users', User),
+ ('profiles', UserProfile),
+ ('teams', Team),
+ ('events', TeamEvent),
+ ('comments', TeamEventComment),
+ ('attendees', Attendee),]
+
+
+class Command(NoArgsCommand):
+ help = "Copy the live data to the local instance"
+
+ def handle_noargs(self, **options):
+ # Delete all old data, so the id's do not conflict when linking
+ print 'Removing local data from database...'
+ for service, Model in ORDER:
+ if Model == User:
+ Model.objects.all().exclude(pk=1).delete() # Do not delete the super user
+ else:
+ Model.objects.all().delete()
+
+ services_url = 'http://loco.ubuntu.com/services/'
+ date_pattern1 = re.compile('^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}$')
+ date_pattern2 = re.compile('^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d+$')
+ id_mapper = {}
+
+ for service, Model in ORDER:
+ id_mapper[Model] = {}
+
+ # Get the json data from the server
+ print 'Getting', service, '...'
+ json_data = urllib2.urlopen(services_url+service).read()
+
+ # Find out what fields are normal, and which many_to_many
+ local_fields = Model._meta.fields
+ many_to_many = Model._meta.many_to_many
+
+ for entry in json.loads(json_data):
+ # Create the model without the many_to_many fields
+ params = {}
+ for field in local_fields:
+ if entry.has_key(field.name) and field.name != 'id':
+ field_value = entry[field.name]
+ if field_value == None:
+ if field.null == True:
+ # If field on the server is None, but None is not allowed, get the default value
+ field_value = field.get_default()
+ else:
+ # If the field is a related, grab the object with that id
+ if field.rel:
+ rel_model = field.rel.to
+ rel_id = id_mapper[rel_model][field_value]
+ try:
+ params[field.name] = rel_model.objects.get(pk=rel_id)
+ except rel_model.DoesNotExist:
+ print '%(model)s with pk=%(pk)s does not exists' % ({'model':rel_model._meta.object_name, 'pk':rel_id})
+ # Else if it is a date, create a date object
+ elif isinstance(field_value, unicode) and date_pattern1.match(field_value):
+ params[field.name] = datetime.datetime.strptime(field_value, '%Y-%m-%d %H:%M:%S')
+ elif isinstance(field_value, unicode) and date_pattern2.match(field_value):
+ params[field.name] = datetime.datetime.strptime(field_value, '%Y-%m-%d %H:%M:%S.%f')
+ # Else: copy the info
+ else:
+ params[field.name] = field_value
+
+ model_instance = Model(**params)
+ model_instance.save()
+
+ # Save the local_pk, and map it to the server_pk
+ id_mapper[Model][entry['id']] = model_instance.id
+
+ # Add the related object to the model
+ for field in many_to_many:
+ if entry.has_key(field.name):
+ for object_id in entry[field.name]:
+ many_related_manager = getattr(model_instance, field.name)
+ rel_model = many_related_manager.model
+ rel_id = id_mapper[rel_model][object_id]
+ try:
+ related_object = rel_model.objects.get(pk=rel_id)
+ except rel_model.DoesNotExist:
+ print '%(model)s with pk=%(pk)s does not exists' % ({'model':rel_model, 'pk':rel_id})
+ else:
+ many_related_manager.add(related_object)
+
+ # Save the model to the database, if changed by many to many fields
+ if many_to_many:
+ model_instance.save()
+
_______________________________________________
Mailing list: https://launchpad.net/~loco-directory-dev
Post to : [email protected]
Unsubscribe : https://launchpad.net/~loco-directory-dev
More help : https://help.launchpad.net/ListHelp
