here's an updated DAL patch to try..... thanks for trudging through this with us!
cfh On Tuesday, February 26, 2013 3:36:30 PM UTC-8, Scott Hunter wrote: > > I'm working from Version 2.3.2 (2012-12-17 15:03:30) stable > > Here's the table in question (I've commented-out the custom_qualifier for > the string fields): > > db.define_table('t_run', > Field('f_trial', type='reference t_trial', > label=T('Trial')), > Field('f_when', type='date', > label=T('When')), > Field('f_dog', type='reference t_dog', > label=T('Dog')), > Field('f_name', type='string',#custom_qualifier={'indexed':False}, > label=T('Name')), > Field('f_breed', type='string',#custom_qualifier={'indexed':False}, > label=T('Breed')), > Field('f_jump_height', type='integer',custom_qualifier={'indexed': > False}, > label=T('Jump Height')), > Field('f_level', type='string',#custom_qualifier={'indexed':False}, > label=T('Level')), > Field('f_class', type='string',#custom_qualifier={'indexed':False}, > label=T('Class')), > Field('f_pref', type='boolean',custom_qualifier={'indexed':False}, > label=T('Preferred')), > Field('f_armband', type='string',#custom_qualifier={'indexed':False}, > label=T('Armband')), > Field('f_yards', type='integer',custom_qualifier={'indexed':False}, > label=T('Yards')), > Field('f_sct', type='integer',custom_qualifier={'indexed':False}, > label=T('SCT')), > Field('f_judge', type='string',#custom_qualifier={'indexed':False}, > label=T('Judge')), > Field('f_score', type='integer',custom_qualifier={'indexed':False}, > label=T('Score')), > Field('f_time', type='double',custom_qualifier={'indexed':False}, > label=T('Time')), > Field('f_faults', type='string',#custom_qualifier={'indexed':False}, > label=T('Faults')), > Field('f_handler', type='string',#custom_qualifier={'indexed':False}, > label=T('Handler')), > Field('f_order', type='integer', > label=T('Order')), > auth.signature, > format='%(f_key)s', > migrate=settings.migrate) > > > P.S. Turning off the indexes DOES make a difference; with the changes > above, instead of using up over 30% of my quota, I'm "only" using 23%. If > I can get the strings unindexed, and take out the web2py-supplied fields > (created_by & _on, modified_by & _on, and maybe is_active), that should get > it down to something manageable. (Not clear on how to handle the > web2py-supplied fields, as I don't know what parameters were using in > making them, making it difficult to know just how to "supply my own > definitions".) > > On Tuesday, February 26, 2013 6:24:21 PM UTC-5, howesc wrote: >> >> your line numbers are off from mine so i'm having trouble making sense of >> this. :( >> >> can you send your model definition so i can see what you are working >> with? i think then i can line it up with the dal version i have here >> (which was trunk from HG as of saturday AM PST) >> >> cfh >> >> On Tuesday, February 26, 2013 3:18:13 PM UTC-8, Scott Hunter wrote: >>> >>> Sorry about that. I've fixed it, and now I get the following: >>> >>> In FILE: /base/data/home/apps/s~sbhweb2py/1.365574604253984974/ >>> applications/ppt_demo/models/db_wizard.py >>> >>> >>> Traceback (most recent call last): >>> File >>> "/base/data/home/apps/s~sbhweb2py/1.365574604253984974/gluon/restricted.py" >>> , line 212, in restricted >>> exec ccode in environment >>> File >>> "/base/data/home/apps/s~sbhweb2py/1.365574604253984974/applications/ppt_demo/models/db_wizard.py" >>> , line 198, in <module> >>> migrate=settings.migrate) >>> File >>> "/base/data/home/apps/s~sbhweb2py/1.365574604253984974/gluon/dal.py",line >>> 7189, in define_table >>> table = self.lazy_define_table(tablename,*fields,**args) >>> File >>> "/base/data/home/apps/s~sbhweb2py/1.365574604253984974/gluon/dal.py",line >>> 7225, in lazy_define_table >>> polymodel=polymodel) >>> File >>> "/base/data/home/apps/s~sbhweb2py/1.365574604253984974/gluon/dal.py",line >>> 4368, in create_table >>> ftype = self.types[field_type](**attr) >>> TypeError: <lambda>() takes no arguments (1 given) >>> >>> >>> This only happens if one of the fields in question is a string; I get no >>> error when the unindexed fields are all integer, boolean or double. >>> >>> - Scott >>> >>> >>> On Tuesday, February 26, 2013 1:04:12 AM UTC-5, howesc wrote: >>>> >>>> it looks like you have a typo.... "custom_qualifer" vs >>>> "custom_qualifier" >>>> >>>> On Monday, February 25, 2013 6:44:14 PM UTC-8, Scott Hunter wrote: >>>>> >>>>> I applied the patch, and added custom_qualifiers like so: >>>>> >>>>> Field('f_name', type='string',custom_qualifer={'indexed':False}, >>>>> label=T('Name')), >>>>> >>>>> >>>>> >>>>> and this is the error I got: >>>>> >>>>> In FILE: /base/data/home/apps/s~sbhweb2py/1.365567821359373728/ >>>>> applications/ppt_demo/models/db_wizard.py >>>>> >>>>> >>>>> Traceback (most recent call last): >>>>> File >>>>> "/base/data/home/apps/s~sbhweb2py/1.365567821359373728/gluon/restricted.py" >>>>> , line 212, in restricted >>>>> exec ccode in environment >>>>> File >>>>> "/base/data/home/apps/s~sbhweb2py/1.365567821359373728/applications/ppt_demo/models/db_wizard.py" >>>>> , line 165, in <module> >>>>> label=T('Name')), >>>>> TypeError: __init__() got an unexpected keyword argument >>>>> 'custom_qualifer' >>>>> >>>>> >>>>> >>>>> >>>>> On Saturday, February 23, 2013 12:30:48 PM UTC-5, howesc wrote: >>>>>> >>>>>> Scott, >>>>>> >>>>>> this is *completely* untested, but here's a proposal: >>>>>> - use the (undocumented) field.custom_qualifier property in GAE >>>>>> field definitions >>>>>> - if you want a field to be unindexed set >>>>>> custom_qualifier={'indexed':False} >>>>>> - for "properties that don't get listed explicitly"....override >>>>>> those default properties with what you want (there are a few techniques >>>>>> for >>>>>> not explicitly listing fields, depending on which technique you are >>>>>> using >>>>>> the answer here is different. >>>>>> >>>>>> this patch against HG trunk this AM might work (i say might cause i >>>>>> have not tested it). are you willing to experiment with it and let us >>>>>> know? >>>>>> >>>>>> thanks, >>>>>> >>>>>> christian >>>>>> >>>>>> On Friday, February 22, 2013 3:53:33 PM UTC-8, Scott Hunter wrote: >>>>>>> >>>>>>> If I had to guess, I'd say a patch is needed in the loop over the >>>>>>> fields in create_table, that would add a new entry to the dict for the >>>>>>> field definition based on a new attribute (which would be ignored for >>>>>>> anything other than Google's Datastore) which gets added to sql_fields; >>>>>>> then migrate_table, which seems to build the table building/altering >>>>>>> commands would need to be made to recognize the new entry & add the >>>>>>> disable >>>>>>> index command. There's not a lot of comments in the code, and I'd be >>>>>>> afraid to break something, but might give it a try. >>>>>>> >>>>>>> - Scott >>>>>>> >>>>>>> P.S. Not sure of the best way to handle this (what's describe above >>>>>>> wouldn't handle it), but it would be nice to be able to turn off the >>>>>>> indexes for the fields that don't get listed explicitly in models.db >>>>>>> (things like created by & when, modified by & when). >>>>>>> >>>>>>> On Friday, February 22, 2013 6:14:28 PM UTC-5, howesc wrote: >>>>>>>> >>>>>>>> thanks for the link. i'll try and take a look this weekend and see >>>>>>>> if there is a place for that in the DAL (feel free to open >>>>>>>> gluon/dal.py >>>>>>>> yourself too). >>>>>>>> >>>>>>>> yup, i'm paying more for writes right now then i am for instance >>>>>>>> hours per day on my largest paid application. :) >>>>>>>> >>>>>>>> cfh >>>>>>>> >>>>>>>> On Thursday, February 21, 2013 5:59:06 PM UTC-8, Scott Hunter wrote: >>>>>>>>> >>>>>>>>> According to >>>>>>>>> https://developers.google.com/appengine/docs/python/datastore/indexes#Unindexed_Properties, >>>>>>>>> >>>>>>>>> "You declare a property unindexed by setting indexed=False in the >>>>>>>>> property >>>>>>>>> constructor". >>>>>>>>> >>>>>>>>> One incurs the cost of a write (or 2?) for every property of a >>>>>>>>> record that gets written; so, for a table with about 25 fields (once >>>>>>>>> you >>>>>>>>> include all of the ones web2py adds), inserting 300 records incurs >>>>>>>>> the cost >>>>>>>>> of 14K+ writes; as the free account has a limit of 50K writes per >>>>>>>>> day, that >>>>>>>>> is rather limiting. It is easy to turn such indicies back on, but >>>>>>>>> you have >>>>>>>>> to re-write every record in order to repopulate them. >>>>>>>>> >>>>>>>>> - Scott >>>>>>>>> >>>>>>>>> On Thursday, February 21, 2013 4:51:59 PM UTC-5, howesc wrote: >>>>>>>>>> >>>>>>>>>> i'm being lazy here....do you have the instructions (or link to >>>>>>>>>> instructions) from GAE for disabling indexes? i don't have it handy >>>>>>>>>> right >>>>>>>>>> now.... >>>>>>>>>> >>>>>>>>>> we can check if the DAL has a secret way to handle it, and/or >>>>>>>>>> create a patch to allow it. >>>>>>>>>> >>>>>>>>>> may i ask what advantage you are hoping to achieve by skipping >>>>>>>>>> those indexes? i've left them on for fear of making a bad choice >>>>>>>>>> that i >>>>>>>>>> can't revert. >>>>>>>>>> >>>>>>>>>> christian >>>>>>>>>> >>>>>>>>>> On Thursday, February 21, 2013 5:56:07 AM UTC-8, Scott Hunter >>>>>>>>>> wrote: >>>>>>>>>>> >>>>>>>>>>> As I understand it, the GAE datastore will automatically create >>>>>>>>>>> an index for every field of every table (to facilitate queries >>>>>>>>>>> using single >>>>>>>>>>> fields); those requiring multiple fields also get generated, but >>>>>>>>>>> will get >>>>>>>>>>> added to index.yaml by the SDK when tested locally. Since these >>>>>>>>>>> are >>>>>>>>>>> created outside of the DAL, I wouldn't expect to be able to remove >>>>>>>>>>> them >>>>>>>>>>> using it. I also understand that there is a way, with the GAE SDK, >>>>>>>>>>> to >>>>>>>>>>> disable the index for any given field (by a parameter in the field >>>>>>>>>>> definition); is there any way to accomplish this through the DAL, >>>>>>>>>>> and if >>>>>>>>>>> so, what is it? >>>>>>>>>> >>>>>>>>>> -- --- You received this message because you are subscribed to the Google Groups "web2py-users" group. To unsubscribe from this group and stop receiving emails from it, send an email to web2py+unsubscr...@googlegroups.com. For more options, visit https://groups.google.com/groups/opt_out.
diff -r 87162e3fcd8c gluon/dal.py --- a/gluon/dal.py Fri Feb 22 18:45:26 2013 -0600 +++ b/gluon/dal.py Tue Feb 26 17:48:11 2013 -0800 @@ -4460,7 +4460,7 @@ adapter_args={}, do_connect=True, after_connection=None): self.types.update({ 'boolean': gae.BooleanProperty, - 'string': (lambda: gae.StringProperty(multiline=True)), + 'string': (lambda **kwargs: gae.StringProperty(multiline=True, **kwargs)), 'text': gae.TextProperty, 'json': gae.TextProperty, 'password': gae.StringProperty, @@ -4476,9 +4476,9 @@ 'datetime': gae.DateTimeProperty, 'id': None, 'reference': gae.IntegerProperty, - 'list:string': (lambda: gae.StringListProperty(default=None)), - 'list:integer': (lambda: gae.ListProperty(int,default=None)), - 'list:reference': (lambda: gae.ListProperty(int,default=None)), + 'list:string': (lambda **kwargs: gae.StringListProperty(default=None, **kwargs)), + 'list:integer': (lambda **kwargs: gae.ListProperty(int,default=None, **kwargs)), + 'list:reference': (lambda **kwargs: gae.ListProperty(int,default=None, **kwargs)), }) self.db = db self.uri = uri @@ -4501,6 +4501,9 @@ if isinstance(polymodel,Table) and field.name in polymodel.fields(): continue attr = {} + if isinstance(field.custom_qaulifier, dict): + #this is custom properties to add to the GAE field declartion + attr = field.custom_qualifier field_type = field.type if isinstance(field_type, SQLCustomType): ftype = self.types[field_type.native or field_type.type](**attr) @@ -4517,10 +4520,10 @@ if field.notnull: attr = dict(required=True) referenced = field_type[10:].strip() - ftype = self.types[field_type[:9]](referenced) + ftype = self.types[field_type[:9]](referenced, **attr) elif field_type.startswith('list:reference'): if field.notnull: - attr = dict(required=True) + attr['required'] = True referenced = field_type[15:].strip() ftype = self.types[field_type[:14]](**attr) elif field_type.startswith('list:'):