I have attached the code doing the write to the database
and a traceback with local variable being display for
each frame.
I am also attaching a run with nothing catching the error
so you can see the simple traceback with the exception.
Here is output from mysql show this has gone a long way
into the file before getting the exception
The code is parsing a XML tree and write records into a database
The records are being buffer and written from the code I have included.
mysql> select count(*) from bids;
+----------+
| count(*) |
+----------+
| 3312460 |
+----------+
1 row in set (0.00 sec)
--~--~---------~--~----~------------~-------~--~----~
You received this message because you are subscribed to the Google Groups
"sqlalchemy" group.
To post to this group, send email to [email protected]
To unsubscribe from this group, send email to [EMAIL PROTECTED]
For more options, visit this group at
http://groups.google.com/group/sqlalchemy?hl=en
-~----------~----~----~----~------~----~------~--~---
#!/usr/bin/env python
# encoding: utf-8
"""
parsingResultsManager.py
Created by David Zaret on 2007-12-29.
Copyright (c) 2007 Symphono, Inc. All rights reserved.
"""
import operator, os, sys
from fairdeal.model import DBSession, engine
from fairdeal.prosper.parsing import mappings
from sqlalchemy.sql import text, bindparam
from datetime import datetime
from fairdeal.prosper.parsing.mappings import columnMappings, TableForTag
class WriteBuffer(object):
"""
Buffer class for write records into the database
This class buffer records to be written into the database
and writes them when max record count rearched.
Records are also flushed when object is deleted
"""
def __init__(self, table):
print "create write buffer %s" % table
self.records = []
self.table_name = table
self.table = TableForTag(table)
self.bufferMax = mappings.MaxValForSetType(table)
self.bufferMax = 4
self.statement = replace(self.table, columnMappings[self.table.name])
def write(self, record):
self.records.append(record)
if len(self.records) >= self.bufferMax:
self.flush()
def flush(self):
if not self.records:
return
print "flushing %s records: %s" % (self.table_name, len(self.records))
conn = engine.connect()
conn.execute(self.statement, self.records)
self.records = []
def replace(table, map):
statement = 'REPLACE INTO %s(%s) VALUES (%s)' % (table.name,
','.join(table.c[key].name for key in table.c.keys()),
','.join(":" + map.get(table.c[key].name, table.c[key].name)
for key in table.c.keys()))
print statement
return text(statement, bindparams=[bindparam(key, type_=table.c[key].type)
for key in table.c.keys()])
**
** FAIRDEAL PYTHON SCRIPT LOGGER - LOG BEGIN **
**
opening file for parsing...
starting parsing...
element start tag ProsperDataExport... we don't care, skipping
create write buffer Bids
REPLACE INTO
bids(LastUpdated,Exchange,Amount,ID,ListingID,ListingStatus,MemberID,MinimumRate,ParticipationAmount,Status)
VALUES
(:LastUpdated,:Exchange,:Amount,:Key,:ListingKey,:ListingStatus,:MemberKey,:MinimumRate,:ParticipationAmount,:Status)
starting parsing of a new set: Bids
flushing Bids records: 4
....
Frame write in
/home/mike/src/optalytic/trunk/src/python/FD/fairdeal/prosper/parsing/parsingResultsManager.py
at line 40
record = Bids
self =
<fairdeal.prosper.parsing.parsingResultsManager.WriteBuffer object at 0xfb8690>
Frame flush in
/home/mike/src/optalytic/trunk/src/python/FD/fairdeal/prosper/parsing/parsingResultsManager.py
at line 48
self =
<fairdeal.prosper.parsing.parsingResultsManager.WriteBuffer object at 0xfb8690>
conn = <sqlalchemy.engine.base.Connection object at
0x104c8d0>
Frame execute in
/usr/lib/python2.5/site-packages/SQLAlchemy-0.4.3-py2.5.egg/sqlalchemy/engine/base.py
at line 846
c = <class
'sqlalchemy.sql.expression.ClauseElement'>
object = REPLACE INTO
bids(LastUpdated,Exchange,Amount,ID,ListingID,ListingStatus,MemberID,MinimumRate,ParticipationAmount,Status)
VALUES
(:LastUpdated,:Exchange,:Amount,:Key,:ListingKey,:ListingStatus,:MemberKey,:MinimumRate,:ParticipationAmount,:Status)
params = {}
multiparams = ([{u'Status': u'Winning', u'CreationDate':
u'2008-01-02T23:38:04', 'Exchange': 1, u'MemberKey':
u'91703368259788903A032A0', u'ListingKey': u'71B734097662498815DDA82',
u'Amount': u'50.00', u'ParticipationAmount': u'50.00', u'ModifiedDate':
u'2008-01-02T23:38:04', u'Key': u'039735130831903594E5AB9', u'ListingStatus':
u'Active'}, {u'Status': u'Winning', u'CreationDate': u'2008-01-02T23:38:05',
'Exchange': 1, u'MemberKey': u'53613375019731860A69F04', u'ListingKey':
u'71B734097662498815DDA82', u'Amount': u'50.00', u'ParticipationAmount':
u'50.00', u'ModifiedDate': u'2008-01-02T23:38:05', u'Key':
u'1E7E3512685856476475D26', u'ListingStatus': u'Active'}, {u'Status':
u'Winning', u'CreationDate': u'2008-01-02T23:38:06', 'Exchange': 1,
u'MemberKey': u'63133403509656034D3A100', u'ListingKey':
u'71B734097662498815DDA82', u'Amount': u'50.00', u'ParticipationAmount':
u'50.00', u'ModifiedDate': u'2008-01-02T23:38:06', u'Key':
u'D2B235137737315033CD9AB', u'ListingStatus': u'Active'}, u'Bids'],)
self = <sqlalchemy.engine.base.Connection object at
0x104c8d0>
Frame execute_clauseelement in
/usr/lib/python2.5/site-packages/SQLAlchemy-0.4.3-py2.5.egg/sqlalchemy/engine/base.py
at line 897
keys = [u'Status', u'CreationDate', 'Exchange',
u'MemberKey', u'ListingKey', u'Amount', u'ParticipationAmount',
u'ModifiedDate', u'Key', u'ListingStatus']
multiparams = ([{u'Status': u'Winning', u'CreationDate':
u'2008-01-02T23:38:04', 'Exchange': 1, u'MemberKey':
u'91703368259788903A032A0', u'ListingKey': u'71B734097662498815DDA82',
u'Amount': u'50.00', u'ParticipationAmount': u'50.00', u'ModifiedDate':
u'2008-01-02T23:38:04', u'Key': u'039735130831903594E5AB9', u'ListingStatus':
u'Active'}, {u'Status': u'Winning', u'CreationDate': u'2008-01-02T23:38:05',
'Exchange': 1, u'MemberKey': u'53613375019731860A69F04', u'ListingKey':
u'71B734097662498815DDA82', u'Amount': u'50.00', u'ParticipationAmount':
u'50.00', u'ModifiedDate': u'2008-01-02T23:38:05', u'Key':
u'1E7E3512685856476475D26', u'ListingStatus': u'Active'}, {u'Status':
u'Winning', u'CreationDate': u'2008-01-02T23:38:06', 'Exchange': 1,
u'MemberKey': u'63133403509656034D3A100', u'ListingKey':
u'71B734097662498815DDA82', u'Amount': u'50.00', u'ParticipationAmount':
u'50.00', u'ModifiedDate': u'2008-01-02T23:38:06', u'Key':
u'D2B235137737315033CD9AB', u'ListingStatus': u'Active'}, u'Bids'],)
params = [{u'Status': u'Winning', u'CreationDate':
u'2008-01-02T23:38:04', 'Exchange': 1, u'MemberKey':
u'91703368259788903A032A0', u'ListingKey': u'71B734097662498815DDA82',
u'Amount': u'50.00', u'ParticipationAmount': u'50.00', u'ModifiedDate':
u'2008-01-02T23:38:04', u'Key': u'039735130831903594E5AB9', u'ListingStatus':
u'Active'}, {u'Status': u'Winning', u'CreationDate': u'2008-01-02T23:38:05',
'Exchange': 1, u'MemberKey': u'53613375019731860A69F04', u'ListingKey':
u'71B734097662498815DDA82', u'Amount': u'50.00', u'ParticipationAmount':
u'50.00', u'ModifiedDate': u'2008-01-02T23:38:05', u'Key':
u'1E7E3512685856476475D26', u'ListingStatus': u'Active'}, {u'Status':
u'Winning', u'CreationDate': u'2008-01-02T23:38:06', 'Exchange': 1,
u'MemberKey': u'63133403509656034D3A100', u'ListingKey':
u'71B734097662498815DDA82', u'Amount': u'50.00', u'ParticipationAmount':
u'50.00', u'ModifiedDate': u'2008-01-02T23:38:06', u'Key':
u'D2B235137737315033CD9AB', u'ListingStatus': u'Active'}, u'Bids']
elem = REPLACE INTO
bids(LastUpdated,Exchange,Amount,ID,ListingID,ListingStatus,MemberID,MinimumRate,ParticipationAmount,Status)
VALUES
(:LastUpdated,:Exchange,:Amount,:Key,:ListingKey,:ListingStatus,:MemberKey,:MinimumRate,:ParticipationAmount,:Status)
self = <sqlalchemy.engine.base.Connection object at
0x104c8d0>
Frame _execute_compiled in
/usr/lib/python2.5/site-packages/SQLAlchemy-0.4.3-py2.5.egg/sqlalchemy/engine/base.py
at line 906
compiled = REPLACE INTO
bids(LastUpdated,Exchange,Amount,ID,ListingID,ListingStatus,MemberID,MinimumRate,ParticipationAmount,Status)
VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)
multiparams = None
params = None
self = <sqlalchemy.engine.base.Connection object at
0x104c8d0>
distilled_params = [{u'Status': u'Winning', u'CreationDate':
u'2008-01-02T23:38:04', 'Exchange': 1, u'MemberKey':
u'91703368259788903A032A0', u'ListingKey': u'71B734097662498815DDA82',
u'Amount': u'50.00', u'ParticipationAmount': u'50.00', u'ModifiedDate':
u'2008-01-02T23:38:04', u'Key': u'039735130831903594E5AB9', u'ListingStatus':
u'Active'}, {u'Status': u'Winning', u'CreationDate': u'2008-01-02T23:38:05',
'Exchange': 1, u'MemberKey': u'53613375019731860A69F04', u'ListingKey':
u'71B734097662498815DDA82', u'Amount': u'50.00', u'ParticipationAmount':
u'50.00', u'ModifiedDate': u'2008-01-02T23:38:05', u'Key':
u'1E7E3512685856476475D26', u'ListingStatus': u'Active'}, {u'Status':
u'Winning', u'CreationDate': u'2008-01-02T23:38:06', 'Exchange': 1,
u'MemberKey': u'63133403509656034D3A100', u'ListingKey':
u'71B734097662498815DDA82', u'Amount': u'50.00', u'ParticipationAmount':
u'50.00', u'ModifiedDate': u'2008-01-02T23:38:06', u'Key':
u'D2B235137737315033CD9AB', u'ListingStatus': u'Active'}, u'Bids']
Frame __create_execution_context in
/usr/lib/python2.5/site-packages/SQLAlchemy-0.4.3-py2.5.egg/sqlalchemy/engine/base.py
at line 953
self = <sqlalchemy.engine.base.Connection object at
0x104c8d0>
e = 'in <string>' requires string as left operand
kwargs = {'compiled':
<sqlalchemy.databases.mysql.MySQLCompiler object at 0x104c610>, 'parameters':
[{u'Status': u'Winning', u'CreationDate': u'2008-01-02T23:38:04', 'Exchange':
1, u'MemberKey': u'91703368259788903A032A0', u'ListingKey':
u'71B734097662498815DDA82', u'Amount': u'50.00', u'ParticipationAmount':
u'50.00', u'ModifiedDate': u'2008-01-02T23:38:04', u'Key':
u'039735130831903594E5AB9', u'ListingStatus': u'Active'}, {u'Status':
u'Winning', u'CreationDate': u'2008-01-02T23:38:05', 'Exchange': 1,
u'MemberKey': u'53613375019731860A69F04', u'ListingKey':
u'71B734097662498815DDA82', u'Amount': u'50.00', u'ParticipationAmount':
u'50.00', u'ModifiedDate': u'2008-01-02T23:38:05', u'Key':
u'1E7E3512685856476475D26', u'ListingStatus': u'Active'}, {u'Status':
u'Winning', u'CreationDate': u'2008-01-02T23:38:06', 'Exchange': 1,
u'MemberKey': u'63133403509656034D3A100', u'ListingKey':
u'71B734097662498815DDA82', u'Amount': u'50.00', u'ParticipationAmount':
u'50.00', u'ModifiedDate': u'2008-01-02T23:38:06', u'Key':
u'D2B235137737315033CD9AB', u'ListingStatus': u'Active'}, u'Bids']}
Frame create_execution_context in
/usr/lib/python2.5/site-packages/SQLAlchemy-0.4.3-py2.5.egg/sqlalchemy/databases/mysql.py
at line 1464
connection = <sqlalchemy.engine.base.Connection object at
0x104c8d0>
self = <sqlalchemy.databases.mysql.MySQLDialect object
at 0xc82d10>
kwargs = {'compiled':
<sqlalchemy.databases.mysql.MySQLCompiler object at 0x104c610>, 'parameters':
[{u'Status': u'Winning', u'CreationDate': u'2008-01-02T23:38:04', 'Exchange':
1, u'MemberKey': u'91703368259788903A032A0', u'ListingKey':
u'71B734097662498815DDA82', u'Amount': u'50.00', u'ParticipationAmount':
u'50.00', u'ModifiedDate': u'2008-01-02T23:38:04', u'Key':
u'039735130831903594E5AB9', u'ListingStatus': u'Active'}, {u'Status':
u'Winning', u'CreationDate': u'2008-01-02T23:38:05', 'Exchange': 1,
u'MemberKey': u'53613375019731860A69F04', u'ListingKey':
u'71B734097662498815DDA82', u'Amount': u'50.00', u'ParticipationAmount':
u'50.00', u'ModifiedDate': u'2008-01-02T23:38:05', u'Key':
u'1E7E3512685856476475D26', u'ListingStatus': u'Active'}, {u'Status':
u'Winning', u'CreationDate': u'2008-01-02T23:38:06', 'Exchange': 1,
u'MemberKey': u'63133403509656034D3A100', u'ListingKey':
u'71B734097662498815DDA82', u'Amount': u'50.00', u'ParticipationAmount':
u'50.00', u'ModifiedDate': u'2008-01-02T23:38:06', u'Key':
u'D2B235137737315033CD9AB', u'ListingStatus': u'Active'}, u'Bids']}
Frame __init__ in
/usr/lib/python2.5/site-packages/SQLAlchemy-0.4.3-py2.5.egg/sqlalchemy/engine/default.py
at line 175
dialect = <sqlalchemy.databases.mysql.MySQLDialect object
at 0xc82d10>
statement = None
parameters = [{u'Status': u'Winning', u'CreationDate':
u'2008-01-02T23:38:04', 'Exchange': 1, u'MemberKey':
u'91703368259788903A032A0', u'ListingKey': u'71B734097662498815DDA82',
u'Amount': u'50.00', u'ParticipationAmount': u'50.00', u'ModifiedDate':
u'2008-01-02T23:38:04', u'Key': u'039735130831903594E5AB9', u'ListingStatus':
u'Active'}, {u'Status': u'Winning', u'CreationDate': u'2008-01-02T23:38:05',
'Exchange': 1, u'MemberKey': u'53613375019731860A69F04', u'ListingKey':
u'71B734097662498815DDA82', u'Amount': u'50.00', u'ParticipationAmount':
u'50.00', u'ModifiedDate': u'2008-01-02T23:38:05', u'Key':
u'1E7E3512685856476475D26', u'ListingStatus': u'Active'}, {u'Status':
u'Winning', u'CreationDate': u'2008-01-02T23:38:06', 'Exchange': 1,
u'MemberKey': u'63133403509656034D3A100', u'ListingKey':
u'71B734097662498815DDA82', u'Amount': u'50.00', u'ParticipationAmount':
u'50.00', u'ModifiedDate': u'2008-01-02T23:38:06', u'Key':
u'D2B235137737315033CD9AB', u'ListingStatus': u'Active'}, u'Bids']
compiled = REPLACE INTO
bids(LastUpdated,Exchange,Amount,ID,ListingID,ListingStatus,MemberID,MinimumRate,ParticipationAmount,Status)
VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)
self =
<sqlalchemy.databases.mysql.MySQLExecutionContext object at 0x104c710>
m = Bids
value = None
connection = <sqlalchemy.engine.base.Connection object at
0x104c8d0>
bindparam = :ParticipationAmount
key = ParticipationAmount
_[3] = [{'Status': u'Winning', 'Exchange': 1,
'MemberKey': u'91703368259788903A032A0', 'LastUpdated': None, 'ListingKey':
u'71B734097662498815DDA82', 'MinimumRate': None, 'Amount': u'50.00',
'ParticipationAmount': u'50.00', 'Key': u'039735130831903594E5AB9',
'ListingStatus': u'Active'}, {'Status': u'Winning', 'Exchange': 1, 'MemberKey':
u'53613375019731860A69F04', 'LastUpdated': None, 'ListingKey':
u'71B734097662498815DDA82', 'MinimumRate': None, 'Amount': u'50.00',
'ParticipationAmount': u'50.00', 'Key': u'1E7E3512685856476475D26',
'ListingStatus': u'Active'}, {'Status': u'Winning', 'Exchange': 1, 'MemberKey':
u'63133403509656034D3A100', 'LastUpdated': None, 'ListingKey':
u'71B734097662498815DDA82', 'MinimumRate': None, 'Amount': u'50.00',
'ParticipationAmount': u'50.00', 'Key': u'D2B235137737315033CD9AB',
'ListingStatus': u'Active'}]
Frame construct_params in
/usr/lib/python2.5/site-packages/SQLAlchemy-0.4.3-py2.5.egg/sqlalchemy/sql/compiler.py
at line 212
name = Status
self = REPLACE INTO
bids(LastUpdated,Exchange,Amount,ID,ListingID,ListingStatus,MemberID,MinimumRate,ParticipationAmount,Status)
VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)
params = Bids
bindparam = :Status
pd = {}
paramname = :Status
Traceback (most recent call last):
File "makedb.py", line 25, in <module>
run()
File "makedb.py", line 21, in run
p.parser.parse(infile)
File "/usr/lib/python2.5/site-packages/_xmlplus/sax/expatreader.py", line
109, in parse
xmlreader.IncrementalParser.parse(self, source)
File "/usr/lib/python2.5/site-packages/_xmlplus/sax/xmlreader.py", line 123,
in parse
self.feed(buffer)
File "/usr/lib/python2.5/site-packages/_xmlplus/sax/expatreader.py", line
216, in feed
self._parser.Parse(data, isFinal)
File "/usr/lib/python2.5/site-packages/_xmlplus/sax/expatreader.py", line
315, in end_element
self._cont_handler.endElement(name)
File
"/home/mike/src/optalytic/trunk/src/python/FD/fairdeal/prosper/parsing/prosperContentHandler.py",
line 79, in endElement
self.buffer.flush()
File
"/home/mike/src/optalytic/trunk/src/python/FD/fairdeal/prosper/parsing/parsingResultsManager.py",
line 48, in flush
conn.execute(self.statement, self.records)
File
"/usr/lib/python2.5/site-packages/SQLAlchemy-0.4.3-py2.5.egg/sqlalchemy/engine/base.py",
line 846, in execute
return Connection.executors[c](self, object, multiparams, params)
File
"/usr/lib/python2.5/site-packages/SQLAlchemy-0.4.3-py2.5.egg/sqlalchemy/engine/base.py",
line 897, in execute_clauseelement
return self._execute_compiled(elem.compile(dialect=self.dialect,
column_keys=keys, inline=len(params) > 1), distilled_params=params)
File
"/usr/lib/python2.5/site-packages/SQLAlchemy-0.4.3-py2.5.egg/sqlalchemy/engine/base.py",
line 906, in _execute_compiled
context = self.__create_execution_context(compiled=compiled,
parameters=distilled_params)
File
"/usr/lib/python2.5/site-packages/SQLAlchemy-0.4.3-py2.5.egg/sqlalchemy/engine/base.py",
line 950, in __create_execution_context
return self.engine.dialect.create_execution_context(connection=self,
**kwargs)
File
"/usr/lib/python2.5/site-packages/SQLAlchemy-0.4.3-py2.5.egg/sqlalchemy/databases/mysql.py",
line 1464, in create_execution_context
return MySQLExecutionContext(self, connection, **kwargs)
File
"/usr/lib/python2.5/site-packages/SQLAlchemy-0.4.3-py2.5.egg/sqlalchemy/engine/default.py",
line 175, in __init__
self.compiled_parameters = [compiled.construct_params(m) for m in
parameters]
File
"/usr/lib/python2.5/site-packages/SQLAlchemy-0.4.3-py2.5.egg/sqlalchemy/sql/compiler.py",
line 212, in construct_params
if paramname in params:
TypeError: 'in <string>' requires string as left operand
