Antony Lesuisse has proposed merging
lp:~openerp-dev/openobject-server/trunk-translationsql-xrg into
lp:openobject-server.
Requested reviews:
OpenERP Core Team (openerp)
For more details, see:
https://code.launchpad.net/~openerp-dev/openobject-server/trunk-translationsql-xrg/+merge/65028
xrg sql loading of translation, much faster.
--
https://code.launchpad.net/~openerp-dev/openobject-server/trunk-translationsql-xrg/+merge/65028
Your team OpenERP R&D Team is subscribed to branch
lp:~openerp-dev/openobject-server/trunk-translationsql-xrg.
=== modified file 'openerp/addons/base/base.sql'
--- openerp/addons/base/base.sql 2011-05-23 13:21:53 +0000
+++ openerp/addons/base/base.sql 2011-06-17 16:09:28 +0000
@@ -314,6 +314,12 @@
primary key(id)
);
+CREATE TABLE res_lang (
+ id serial PRIMARY KEY,
+ name VARCHAR(64) NOT NULL UNIQUE,
+ code VARCHAR(16) NOT NULL UNIQUE
+);
+
CREATE TABLE ir_model_data (
id serial NOT NULL,
create_uid integer,
=== modified file 'openerp/addons/base/ir/ir_translation.py'
--- openerp/addons/base/ir/ir_translation.py 2011-06-08 14:50:33 +0000
+++ openerp/addons/base/ir/ir_translation.py 2011-06-17 16:09:28 +0000
@@ -21,6 +21,7 @@
from osv import fields, osv
import tools
+import logging
TRANSLATION_TYPE = [
('field', 'Field'),
@@ -39,6 +40,115 @@
('sql_constraint', 'SQL Constraint')
]
+class ir_translation_import_cursor(object):
+ """Temporary cursor for optimizing mass insert into ir.translation
+
+ Open it (attached to a sql cursor), feed it with translation data and
+ finish() it in order to insert multiple translations in a batch.
+ """
+ _table_name = 'tmp_ir_translation_import'
+
+ def __init__(self, cr, uid, parent, context):
+ """ Initializer
+
+ Store some values, and also create a temporary SQL table to accept
+ the data.
+ @param parent an instance of ir.translation ORM model
+ """
+
+ self._cr = cr
+ self._uid = uid
+ self._context = context
+ self._overwrite = context.get('overwrite', False)
+ self._debug = False
+ self._parent_table = parent._table
+
+ # Note that Postgres will NOT inherit the constraints or indexes
+ # of ir_translation, so this copy will be much faster.
+
+ cr.execute('''CREATE TEMP TABLE %s(
+ imd_model VARCHAR(64),
+ imd_module VARCHAR(64),
+ imd_name VARCHAR(128)
+ ) INHERITS (%s) ''' % (self._table_name, self._parent_table))
+
+ def push(self, ddict):
+ """Feed a translation, as a dictionary, into the cursor
+ """
+
+ self._cr.execute("INSERT INTO " + self._table_name \
+ + """(name, lang, res_id, src, type,
+ imd_model, imd_module, imd_name, value)
+ VALUES(%s, %s, %s, %s, %s, %s, %s, %s, %s)""",
+ (ddict['name'], ddict['lang'], ddict.get('res_id'), ddict['src'], ddict['type'],
+ ddict.get('imd_model'), ddict.get('imd_module'), ddict.get('imd_name'),
+ ddict['value']))
+
+ def finish(self):
+ """ Transfer the data from the temp table to ir.translation
+ """
+ logger = logging.getLogger('orm')
+
+ cr = self._cr
+ if self._debug:
+ cr.execute("SELECT count(*) FROM %s" % self._table_name)
+ c = cr.fetchone()[0]
+ logger.debug("ir.translation.cursor: We have %d entries to process", c)
+
+ # Step 1: resolve ir.model.data references to res_ids
+ cr.execute("""UPDATE %s AS ti
+ SET res_id = imd.res_id
+ FROM ir_model_data AS imd
+ WHERE ti.res_id IS NULL
+ AND ti.imd_module IS NOT NULL AND ti.imd_name IS NOT NULL
+
+ AND ti.imd_module = imd.module AND ti.imd_name = imd.name
+ AND ti.imd_model = imd.model; """ % self._table_name)
+
+ if self._debug:
+ cr.execute("SELECT imd_module, imd_model, imd_name FROM %s " \
+ "WHERE res_id IS NULL AND imd_module IS NOT NULL" % self._table_name)
+ for row in cr.fetchall():
+ logger.debug("ir.translation.cursor: missing res_id for %s. %s/%s ", *row)
+
+ cr.execute("DELETE FROM %s WHERE res_id IS NULL AND imd_module IS NOT NULL" % \
+ self._table_name)
+
+ # Records w/o res_id must _not_ be inserted into our db, because they are
+ # referencing non-existent data.
+
+ find_expr = "irt.lang = ti.lang AND irt.type = ti.type " \
+ " AND irt.name = ti.name AND irt.src = ti.src " \
+ " AND (ti.type != 'model' OR ti.res_id = irt.res_id) "
+
+ # Step 2: update existing (matching) translations
+ if self._overwrite:
+ cr.execute("""UPDATE ONLY %s AS irt
+ SET value = ti.value
+ FROM %s AS ti
+ WHERE %s AND ti.value IS NOT NULL AND ti.value != ''
+ """ % (self._parent_table, self._table_name, find_expr))
+
+ # Step 3: insert new translations
+
+ cr.execute("""INSERT INTO %s(name, lang, res_id, src, type, value)
+ SELECT name, lang, res_id, src, type, value
+ FROM %s AS ti
+ WHERE NOT EXISTS(SELECT 1 FROM ONLY %s AS irt WHERE %s);
+ """ % (self._parent_table, self._table_name, self._parent_table, find_expr))
+
+ if self._debug:
+ cr.execute('SELECT COUNT(*) FROM ONLY %s' % (self._parent_table))
+ c1 = cr.fetchone()[0]
+ cr.execute('SELECT COUNT(*) FROM ONLY %s AS irt, %s AS ti WHERE %s' % \
+ (self._parent_table, self._table_name, find_expr))
+ c = cr.fetchone()[0]
+ logger.debug("ir.translation.cursor: %d entries now in ir.translation, %d common entries with tmp", c1, c)
+
+ # Step 4: cleanup
+ cr.execute("DROP TABLE %s" % self._table_name)
+ return True
+
class ir_translation(osv.osv):
_name = "ir.translation"
_log_access = False
@@ -56,13 +166,13 @@
'type': fields.selection(TRANSLATION_TYPE, string='Type', size=16, select=True),
'src': fields.text('Source'),
'value': fields.text('Translation Value'),
- # These two columns map to ir_model_data.module and ir_model_data.name.
- # They are used to resolve the res_id above after loading is done.
- 'module': fields.char('Module', size=64, help='Maps to the ir_model_data for which this translation is provided.'),
- 'xml_id': fields.char('XML Id', size=128, help='Maps to the ir_model_data for which this translation is provided.'),
}
-
- def _auto_init(self, cr, context={}):
+
+ _sql_constraints = [ ('lang_fkey_res_lang', 'FOREIGN KEY(lang) REFERENCES res_lang(code)',
+ 'Language code of translation item must be among known languages' ), ]
+
+
+ def _auto_init(self, cr, context=None):
super(ir_translation, self)._auto_init(cr, context)
# FIXME: there is a size limit on btree indexed values so we can't index src column with normal btree.
@@ -87,6 +197,11 @@
cr.execute('CREATE INDEX ir_translation_ltn ON ir_translation (name, lang, type)')
cr.commit()
+ def _check_selection_field_value(self, cr, uid, field, value, context=None):
+ if field == 'lang':
+ return
+ return super(ir_translation, self)._check_selection_field_value(cr, uid, field, value, context=context)
+
@tools.cache(skiparg=3, multi='ids')
def _get_ids(self, cr, uid, name, tt, lang, ids):
translations = dict.fromkeys(ids, False)
@@ -204,6 +319,13 @@
result = super(ir_translation, self).unlink(cursor, user, ids, context=context)
return result
+ def _get_import_cursor(self, cr, uid, context=None):
+ """ Return a cursor-like object for fast inserting translations
+ """
+ if context is None:
+ context = {}
+ return ir_translation_import_cursor(cr, uid, self, context=context)
+
ir_translation()
# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:
=== modified file 'openerp/addons/base/module/module.py'
--- openerp/addons/base/module/module.py 2011-05-11 13:53:32 +0000
+++ openerp/addons/base/module/module.py 2011-06-17 16:09:28 +0000
@@ -519,7 +519,7 @@
tools.trans_load(cr, f, lang, verbose=False, context=context2)
elif iso_lang != 'en':
logger.warning('module %s: no translation for language %s', mod.name, iso_lang)
- tools.trans_update_res_ids(cr)
+ # tools.trans_update_res_ids(cr)
def check(self, cr, uid, ids, context=None):
logger = logging.getLogger('init')
=== modified file 'openerp/tools/translate.py'
--- openerp/tools/translate.py 2011-06-14 14:22:26 +0000
+++ openerp/tools/translate.py 2011-06-17 16:09:28 +0000
@@ -891,6 +891,8 @@
# read the rest of the file
line = 1
+ irt_cursor = trans_obj._get_import_cursor(cr, uid, context=context)
+
for row in reader:
line += 1
# skip empty rows and rows where the translation field (=last fiefd) is empty
@@ -901,57 +903,47 @@
# {'lang': ..., 'type': ..., 'name': ..., 'res_id': ...,
# 'src': ..., 'value': ...}
dic = {'lang': lang}
+ dic_module = False
for i in range(len(f)):
if f[i] in ('module',):
continue
dic[f[i]] = row[i]
- try:
- dic['res_id'] = dic['res_id'] and int(dic['res_id']) or 0
- dic['module'] = False
- dic['xml_id'] = False
- except:
- split_id = dic['res_id'].split('.', 1)
- dic['module'] = split_id[0]
- dic['xml_id'] = split_id[1]
- dic['res_id'] = False
+ # This would skip terms that fail to specify a res_id
+ if not dic.get('res_id', False):
+ continue
- args = [
- ('lang', '=', lang),
- ('type', '=', dic['type']),
- ('name', '=', dic['name']),
- ('src', '=', dic['src']),
- ]
- if dic['type'] == 'model':
- if dic['res_id'] is False:
- args.append(('module', '=', dic['module']))
- args.append(('xml_id', '=', dic['xml_id']))
- else:
- args.append(('res_id', '=', dic['res_id']))
- ids = trans_obj.search(cr, uid, args)
- if ids:
- if context.get('overwrite') and dic['value']:
- trans_obj.write(cr, uid, ids, {'value': dic['value']})
+ res_id = dic.pop('res_id')
+ if res_id and isinstance(res_id, (int, long)) \
+ or (isinstance(res_id, basestring) and res_id.isdigit()):
+ dic['res_id'] = int(res_id)
else:
- trans_obj.create(cr, uid, dic)
+ try:
+ tmodel = dic['name'].split(',')[0]
+ if '.' in res_id:
+ tmodule, tname = res_id.split('.', 1)
+ else:
+ tmodule = dic_module
+ tname = res_id
+ dic['imd_model'] = tmodel
+ dic['imd_module'] = tmodule
+ dic['imd_name'] = tname
+
+ dic['res_id'] = None
+ except Exception:
+ logger.warning("Could not decode resource for %s, please fix the po file.",
+ dic['res_id'], exc_info=True)
+ dic['res_id'] = None
+
+ irt_cursor.push(dic)
+
+ irt_cursor.finish()
if verbose:
logger.info("translation file loaded succesfully")
except IOError:
filename = '[lang: %s][format: %s]' % (iso_lang or 'new', fileformat)
logger.exception("couldn't read translation file %s", filename)
-def trans_update_res_ids(cr):
- cr.execute("""
- UPDATE ir_translation
- SET res_id = COALESCE ((SELECT ir_model_data.res_id
- FROM ir_model_data
- WHERE ir_translation.module = ir_model_data.module
- AND ir_translation.xml_id = ir_model_data.name), 0)
- WHERE ir_translation.module is not null
- AND ir_translation.xml_id is not null
- AND ir_translation.res_id = 0;
- """)
-
def get_locales(lang=None):
if lang is None:
lang = locale.getdefaultlocale()[0]
_______________________________________________
Mailing list: https://launchpad.net/~openerp-dev-gtk
Post to : [email protected]
Unsubscribe : https://launchpad.net/~openerp-dev-gtk
More help : https://help.launchpad.net/ListHelp
