Thanks Ikai again for the fast reply.
I tried the following
('brand', lambda x: x.decode('utf-8'), ""),
so changing the default value from None to "" but that didn't help
Then tried the following
('brand', if x is not None: lambda x: x.decode('utf-8'), ""),
but this threw a syntax error.
Is there anyway to check that the value is not null in order to decode
it?
Thanks
Dominik
> It's basically the equivalent of a Null Pointer exception. This
> syntax:
>
> lambda x: x.someMethod()
>
> Is an inline anonymous function call. If x is None as opposed to
> blank String, you will not be able to call methods on it.
>
> On Thu, Dec 10, 2009 at 1:45 PM, Dominik Steiner
> <[email protected]
> > wrote:
> Thanks Ikai for the fast response,
>
> yes, some unicode characters might be the issue here as my data is
> in Spanish. I tried your suggestion and now have an exporter.py that
> looks like this
>
> from google.appengine.ext import db
>
> class Product(db.Model):
> userId = db.StringProperty()
> name = db.StringProperty()
> shop = db.StringProperty()
> brand = db.StringProperty()
> model = db.StringProperty()
> contact = db.StringProperty()
> imageId = db.StringProperty()
> userNickName = db.StringProperty()
> price = db.FloatProperty()
> registeredDate = db.DateProperty()
> validUntilDate = db.DateProperty()
> ratingCount = db.IntegerProperty()
> ratingTotal = db.IntegerProperty()
> rating = db.IntegerProperty()
> commentCount = db.IntegerProperty()
> description = db.TextProperty()
>
> from google.appengine.ext import db
> from google.appengine.tools import bulkloader
>
> class ProductExporter(bulkloader.Exporter):
> def __init__(self):
> bulkloader.Exporter.__init__(self, 'Product',
> [('name', lambda x:
> x.decode('utf-8'), None),
> ('userId', str, None),
> ('shop', lambda x:
> x.decode('utf-8'), None).
> ('brand', lambda x:
> x.decode('utf-8'), None),
> ('model', lambda x:
> x.decode('utf-8'), None),
> ('contact', lambda x:
> x.decode('utf-8'), None),
> ('imageId', str, None),
> ('userNickName', lambda x:
> x.decode('utf-8'), None),
> ('price', str, None),
> ('registeredDate', str, None),
> ('validUntilDate', str, None),
> ('ratingCount', str, None),
> ('ratingTotal', str, None),
> ('rating', str, None),
> ('commentCount', str, None),
> ('description', lambda x:
> x.decode('utf-8'), None)
> ])
> exporters = [ProductExporter]
>
>
> and I now get the following error (but at least it seems that it had
> been downloading for a while before throwing that error)
>
> Traceback (most recent call last):
> File "/usr/local/bin/appcfg.py", line 60, in <module>
> run_file(__file__, globals())
> File "/usr/local/bin/appcfg.py", line 57, in run_file
> execfile(script_path, globals_)
> File "/Applications/GoogleAppEngineLauncher.app/Contents/Resources/
> GoogleAppEngine-default.bundle/Contents/Resources/google_appengine/
> google/appengine/tools/appcfg.py", line 2548, in <module>
> main(sys.argv)
> File "/Applications/GoogleAppEngineLauncher.app/Contents/Resources/
> GoogleAppEngine-default.bundle/Contents/Resources/google_appengine/
> google/appengine/tools/appcfg.py", line 2539, in main
> result = AppCfgApp(argv).Run()
> File "/Applications/GoogleAppEngineLauncher.app/Contents/Resources/
> GoogleAppEngine-default.bundle/Contents/Resources/google_appengine/
> google/appengine/tools/appcfg.py", line 1640, in Run
> self.action(self)
> File "/Applications/GoogleAppEngineLauncher.app/Contents/Resources/
> GoogleAppEngine-default.bundle/Contents/Resources/google_appengine/
> google/appengine/tools/appcfg.py", line 2427, in __call__
> return method()
> File "/Applications/GoogleAppEngineLauncher.app/Contents/Resources/
> GoogleAppEngine-default.bundle/Contents/Resources/google_appengine/
> google/appengine/tools/appcfg.py", line 2293, in PerformDownload
> run_fn(args)
> File "/Applications/GoogleAppEngineLauncher.app/Contents/Resources/
> GoogleAppEngine-default.bundle/Contents/Resources/google_appengine/
> google/appengine/tools/appcfg.py", line 2215, in RunBulkloader
> sys.exit(bulkloader.Run(arg_dict))
> File "/Applications/GoogleAppEngineLauncher.app/Contents/Resources/
> GoogleAppEngine-default.bundle/Contents/Resources/google_appengine/
> google/appengine/tools/bulkloader.py", line 3894, in Run
> return _PerformBulkload(arg_dict)
> File "/Applications/GoogleAppEngineLauncher.app/Contents/Resources/
> GoogleAppEngine-default.bundle/Contents/Resources/google_appengine/
> google/appengine/tools/bulkloader.py", line 3800, in _PerformBulkload
> return_code = app.Run()
> File "/Applications/GoogleAppEngineLauncher.app/Contents/Resources/
> GoogleAppEngine-default.bundle/Contents/Resources/google_appengine/
> google/appengine/tools/bulkloader.py", line 3158, in Run
> self.progress_thread.WorkFinished()
> File "/Applications/GoogleAppEngineLauncher.app/Contents/Resources/
> GoogleAppEngine-default.bundle/Contents/Resources/google_appengine/
> google/appengine/tools/bulkloader.py", line 2207, in WorkFinished
> exporter.output_entities(self.result_db.AllEntities())
> File "/Applications/GoogleAppEngineLauncher.app/Contents/Resources/
> GoogleAppEngine-default.bundle/Contents/Resources/google_appengine/
> google/appengine/tools/bulkloader.py", line 2788, in output_entities
> for entity in entity_generator)
> File "/Applications/GoogleAppEngineLauncher.app/Contents/Resources/
> GoogleAppEngine-default.bundle/Contents/Resources/google_appengine/
> google/appengine/tools/bulkloader.py", line 2788, in <genexpr>
> for entity in entity_generator)
> File "/Applications/GoogleAppEngineLauncher.app/Contents/Resources/
> GoogleAppEngine-default.bundle/Contents/Resources/google_appengine/
> google/appengine/tools/bulkloader.py", line 2769, in __SerializeEntity
> encoding = self.__EncodeEntity(entity)
> File "/Applications/GoogleAppEngineLauncher.app/Contents/Resources/
> GoogleAppEngine-default.bundle/Contents/Resources/google_appengine/
> google/appengine/tools/bulkloader.py", line 2757, in __EncodeEntity
> writer.writerow(self.__ExtractProperties(entity))
> File "/Applications/GoogleAppEngineLauncher.app/Contents/Resources/
> GoogleAppEngine-default.bundle/Contents/Resources/google_appengine/
> google/appengine/tools/bulkloader.py", line 2738, in
> __ExtractProperties
> encoding.append(fn(entity[name]))
> File "exporter.py", line 30, in <lambda>
> ('brand', lambda x: x.decode('utf-8'), None),
> AttributeError: 'NoneType' object has no attribute 'decode'
>
> Do you have any idea of what that could be?
>
> Thanks again for your help.
>
> Dominik
>> Interesting. Do you have some unicode characters in any fields?
>> There's a snippet on the bulk loader page about how to handle these
>> characters in your Importer:
>>
>> http://code.google.com/appengine/docs/python/tools/uploadingdata.html
>>
>> import datetime
>> from google.appengine.ext import db
>> from google.appengine.tools import bulkloader
>> import models
>>
>> class AlbumLoader(bulkloader.Loader):
>> def __init__(self):
>> bulkloader.Loader.__init__(self, 'Album',
>> [('title', lambda x:
>> x.decode('utf-8')),
>> ('artist', lambda x:
>> x.decode('utf-8')),
>> ('publication_date',
>> lambda x:
>> datetime.datetime.strptime(x, '%m/%d/%Y').date()),
>> ('length_in_minutes', int)
>> ])
>>
>> loaders = [AlbumLoader]
>>
>> Your error may be caused by UTF-8/ASCII incompatibilities, and this
>> is where I would start looking.
>>
>> On Wed, Dec 9, 2009 at 8:26 PM, Dominik Steiner
>> <[email protected]
>> > wrote:
>> Hi Ikai,
>>
>> I followed the instructions in the cook book for the python uploader/
>> downloader in order to be able to download and backup the data of my
>> java application.
>>
>> As already posted in the cook book article, I got the following
>> situation:
>>
>> "I would love to be able to download data in order to backup the data
>> of my application and I tried your steps. One thing that didn't work
>> for me was the line
>>
>> appcfg.py --server=python.latest.appid.appspot.com download_data
>> exporter --filename=data.csv --kind=Thing --config_file=exporter/
>> thing_exporter.py
>>
>> but had to write
>>
>> appcfg.py --server=python.latest.xelavos.appspot.com download_data --
>> filename=data.csv --kind=Thing --config_file=exporter/
>> thing_exporter.py path/to/my/pythondirectory
>>
>> Then after being able to launch the exporter i got the following
>> error
>> after a while of downloading
>>
>> .[INFO ] Product: No descending index on __key__, performing serial
>> download
>> .........................................
>> Traceback (most recent call last):
>> File "/usr/local/bin/appcfg.py", line 60, in
>> run_file(__file__, globals())
>> File "/usr/local/bin/appcfg.py", line 57, in run_file
>> execfile(script_path, globals_)
>> File "/Applications/GoogleAppEngineLauncher.app/Contents/Resources/
>> GoogleAppEngine-default.bundle/Contents/Resources/google_appengine/
>> google/appengine/tools/appcfg.py", line 2548, in
>> main(sys.argv)
>> File "/Applications/GoogleAppEngineLauncher.app/Contents/Resources/
>> GoogleAppEngine-default.bundle/Contents/Resources/google_appengine/
>> google/appengine/tools/appcfg.py", line 2539, in main
>> result = AppCfgApp(argv).Run()
>> File "/Applications/GoogleAppEngineLauncher.app/Contents/Resources/
>> GoogleAppEngine-default.bundle/Contents/Resources/google_appengine/
>> google/appengine/tools/appcfg.py", line 1640, in Run
>> self.action(self)
>> File "/Applications/GoogleAppEngineLauncher.app/Contents/Resources/
>> GoogleAppEngine-default.bundle/Contents/Resources/google_appengine/
>> google/appengine/tools/appcfg.py", line 2427, in __call__
>> return method()
>> File "/Applications/GoogleAppEngineLauncher.app/Contents/Resources/
>> GoogleAppEngine-default.bundle/Contents/Resources/google_appengine/
>> google/appengine/tools/appcfg.py", line 2293, in PerformDownload
>> run_fn(args)
>> File "/Applications/GoogleAppEngineLauncher.app/Contents/Resources/
>> GoogleAppEngine-default.bundle/Contents/Resources/google_appengine/
>> google/appengine/tools/appcfg.py", line 2215, in RunBulkloader
>> sys.exit(bulkloader.Run(arg_dict))
>> File "/Applications/GoogleAppEngineLauncher.app/Contents/Resources/
>> GoogleAppEngine-default.bundle/Contents/Resources/google_appengine/
>> google/appengine/tools/bulkloader.py", line 3894, in Run
>> return _PerformBulkload(arg_dict)
>> File "/Applications/GoogleAppEngineLauncher.app/Contents/Resources/
>> GoogleAppEngine-default.bundle/Contents/Resources/google_appengine/
>> google/appengine/tools/bulkloader.py", line 3800, in _PerformBulkload
>> return_code = app.Run()
>> File "/Applications/GoogleAppEngineLauncher.app/Contents/Resources/
>> GoogleAppEngine-default.bundle/Contents/Resources/google_appengine/
>> google/appengine/tools/bulkloader.py", line 3158, in Run
>> self.progress_thread.WorkFinished()
>> File "/Applications/GoogleAppEngineLauncher.app/Contents/Resources/
>> GoogleAppEngine-default.bundle/Contents/Resources/google_appengine/
>> google/appengine/tools/bulkloader.py", line 2207, in WorkFinished
>> exporter.output_entities(self.result_db.AllEntities())
>> File "/Applications/GoogleAppEngineLauncher.app/Contents/Resources/
>> GoogleAppEngine-default.bundle/Contents/Resources/google_appengine/
>> google/appengine/tools/bulkloader.py", line 2788, in output_entities
>> for entity in entity_generator)
>> File "/Applications/GoogleAppEngineLauncher.app/Contents/Resources/
>> GoogleAppEngine-default.bundle/Contents/Resources/google_appengine/
>> google/appengine/tools/bulkloader.py", line 2788, in
>> for entity in entity_generator)
>> File "/Applications/GoogleAppEngineLauncher.app/Contents/Resources/
>> GoogleAppEngine-default.bundle/Contents/Resources/google_appengine/
>> google/appengine/tools/bulkloader.py", line 2769, in
>> __SerializeEntity
>> encoding = self.__EncodeEntity(entity)
>> File "/Applications/GoogleAppEngineLauncher.app/Contents/Resources/
>> GoogleAppEngine-default.bundle/Contents/Resources/google_appengine/
>> google/appengine/tools/bulkloader.py", line 2757, in __EncodeEntity
>> writer.writerow(self.__ExtractProperties(entity))
>> File "/Applications/GoogleAppEngineLauncher.app/Contents/Resources/
>> GoogleAppEngine-default.bundle/Contents/Resources/google_appengine/
>> google/appengine/tools/bulkloader.py", line 2738, in
>> __ExtractProperties
>> encoding.append(fn(entity[name]))
>> UnicodeEncodeError: 'ascii' codec can't encode character u'\xe1' in
>> position 7: ordinal not in range(128)
>> "
>>
>> Do you know what this error means and how i can fix it?
>>
>> Thanks for any help
>>
>> Dominik
>>
>> --
>>
>> You received this message because you are subscribed to the Google
>> Groups "Google App Engine for Java" group.
>> To post to this group, send email to [email protected]
>> .
>> To unsubscribe from this group, send email to
>> [email protected]
>> .
>> For more options, visit this group at
>> http://groups.google.com/group/google-appengine-java?hl=en
>> .
>>
>>
>>
>>
>>
>> --
>> Ikai Lan
>> Developer Programs Engineer, Google App Engine
>>
>> --
>>
>> You received this message because you are subscribed to the Google
>> Groups "Google App Engine for Java" group.
>> To post to this group, send email to [email protected]
>> .
>> To unsubscribe from this group, send email to
>> [email protected]
>> .
>> For more options, visit this group at
>> http://groups.google.com/group/google-appengine-java?hl=en
>> .
>
>
> --
>
> You received this message because you are subscribed to the Google
> Groups "Google App Engine for Java" group.
> To post to this group, send email to [email protected]
> .
> To unsubscribe from this group, send email to
> [email protected]
> .
> For more options, visit this group at
> http://groups.google.com/group/google-appengine-java?hl=en
> .
>
>
>
> --
> Ikai Lan
> Developer Programs Engineer, Google App Engine
>
> --
>
> You received this message because you are subscribed to the Google
> Groups "Google App Engine for Java" group.
> To post to this group, send email to [email protected]
> .
> To unsubscribe from this group, send email to
> [email protected]
> .
> For more options, visit this group at
> http://groups.google.com/group/google-appengine-java?hl=en
> .
--
You received this message because you are subscribed to the Google Groups
"Google App Engine for Java" group.
To post to this group, send email to [email protected].
To unsubscribe from this group, send email to
[email protected].
For more options, visit this group at
http://groups.google.com/group/google-appengine-java?hl=en.
