Author: matevz
Date: Wed Jul 24 10:56:43 2013
New Revision: 1506493
URL: http://svn.apache.org/r1506493
Log:
alternative 1 implementations + tests
Modified:
bloodhound/branches/bep_0010_ticket_numbering/bloodhound_multiproduct/multiproduct/api.py
bloodhound/branches/bep_0010_ticket_numbering/bloodhound_multiproduct/multiproduct/dbcursor.py
bloodhound/branches/bep_0010_ticket_numbering/bloodhound_multiproduct/multiproduct/util.py
bloodhound/branches/bep_0010_ticket_numbering/bloodhound_multiproduct/tests/db/cursor.py
bloodhound/branches/bep_0010_ticket_numbering/bloodhound_multiproduct/tests/ticket/model.py
bloodhound/branches/bep_0010_ticket_numbering/bloodhound_multiproduct/tests/upgrade.py
bloodhound/branches/bep_0010_ticket_numbering/trac/trac/ticket/model.py
Modified:
bloodhound/branches/bep_0010_ticket_numbering/bloodhound_multiproduct/multiproduct/api.py
URL:
http://svn.apache.org/viewvc/bloodhound/branches/bep_0010_ticket_numbering/bloodhound_multiproduct/multiproduct/api.py?rev=1506493&r1=1506492&r2=1506493&view=diff
==============================================================================
---
bloodhound/branches/bep_0010_ticket_numbering/bloodhound_multiproduct/multiproduct/api.py
(original)
+++
bloodhound/branches/bep_0010_ticket_numbering/bloodhound_multiproduct/multiproduct/api.py
Wed Jul 24 10:56:43 2013
@@ -45,11 +45,12 @@ from trac.wiki.parser import WikiParser
from multiproduct.dbcursor import GLOBAL_PRODUCT
from multiproduct.model import Product, ProductResourceMap, ProductSetting
-from multiproduct.util import EmbeddedLinkFormatter, IDENTIFIER
+from multiproduct.util import EmbeddedLinkFormatter, IDENTIFIER, \
+ using_sqlite_backend
__all__ = ['MultiProductSystem', 'PRODUCT_SYNTAX_DELIMITER']
-DB_VERSION = 4
+DB_VERSION = 5
DB_SYSTEM_KEY = 'bloodhound_multi_product_version'
PLUGIN_NAME = 'Bloodhound multi product'
@@ -242,6 +243,12 @@ class MultiProductSystem(Component):
self._create_product_tables_for_plugins(db)
db_installed_version = self._update_db_version(db, 4)
+ if db_installed_version < 5:
+ table_defs = self._add_product_column_to_tables(
+ ['ticket'], db_installed_version)
+ self._modify_ticket_pk(db, table_defs)
+ db_installed_version = self._update_db_version(db, 5)
+
self.env.enable_multiproduct_schema(True)
def _add_column_product_to_ticket(self, db):
@@ -302,7 +309,7 @@ class MultiProductSystem(Component):
(t.name, t.key, PLUGIN_NAME, current_version, 3))
return table_defs
- def _get_table_columns(self, table_defs):
+ def _get_table_columns(self, table_defs, all_columns=False):
table_columns = dict()
for table in table_defs:
table_definition = \
@@ -310,7 +317,7 @@ class MultiProductSystem(Component):
column_names = \
[column.name for column in table_definition.columns]
table_columns[table.name] = \
- [c for c in column_names if c != 'product']
+ [c for c in column_names if all_columns or c != 'product']
return table_columns
def _insert_default_product(self, db):
@@ -512,6 +519,39 @@ class MultiProductSystem(Component):
for statement in db_connector.to_sql(ProductSetting._get_schema()):
db(statement)
+ def _modify_ticket_pk(self, db, table_defs):
+ self.log.debug("Modifying ticket primary key: id -> uid")
+ table_columns = self._get_table_columns(table_defs, True)
+ db_connector, _ = DatabaseManager(self.env)._get_connector()
+
+ def rename_id_to_uid(table):
+ for c in table.columns:
+ if c.name == 'id':
+ c.name = 'uid'
+ break
+ table.key = ['uid']
+
+ def add_new_id_column(table):
+ id_column = Column('id', type='int', auto_increment=True)
+ if using_sqlite_backend(self.env):
+ # sqlite does not support multiple auto increment columns
+ id_column.auto_increment = False
+ table.columns.append(id_column)
+
+
+ for t in table_defs:
+ rename_id_to_uid(t)
+ add_new_id_column(t)
+
+ temp_table_name, cols = self._create_temp_table(
+ db, t.name, table_columns, table_defs)
+ db("""INSERT INTO ticket (%s, uid)
+ SELECT %s, id FROM ticket_temp""" %
+ (cols, cols))
+ self._drop_temp_table(db, temp_table_name)
+ db.update_sequence(db.cursor(), 'ticket', 'id')
+ db.update_sequence(db.cursor(), 'ticket', 'uid')
+
# IResourceChangeListener methods
def match_resource(self, resource):
return isinstance(resource, Product)
Modified:
bloodhound/branches/bep_0010_ticket_numbering/bloodhound_multiproduct/multiproduct/dbcursor.py
URL:
http://svn.apache.org/viewvc/bloodhound/branches/bep_0010_ticket_numbering/bloodhound_multiproduct/multiproduct/dbcursor.py?rev=1506493&r1=1506492&r2=1506493&view=diff
==============================================================================
---
bloodhound/branches/bep_0010_ticket_numbering/bloodhound_multiproduct/multiproduct/dbcursor.py
(original)
+++
bloodhound/branches/bep_0010_ticket_numbering/bloodhound_multiproduct/multiproduct/dbcursor.py
Wed Jul 24 10:56:43 2013
@@ -24,6 +24,7 @@ import sqlparse.tokens as Tokens
import sqlparse.sql as Types
from multiproduct.cache import lru_cache
+from multiproduct.util import using_sqlite_backend
__all__ = ['BloodhoundIterableCursor', 'BloodhoundConnectionWrapper',
'ProductEnvContextManager']
@@ -41,7 +42,7 @@ TRANSLATE_TABLES = ['system',
'permission',
'wiki',
'report',
- ]
+ ]
PRODUCT_COLUMN = 'product'
GLOBAL_PRODUCT = ''
@@ -55,21 +56,22 @@ translator_not_set = empty_translator()
def translate_sql(env, sql):
translator = None
log = None
+ product_prefix = None
if env is not None:
- # FIXME: This is the right way to do it but breaks translation
- # if trac.db.api.DatabaseManager(self.env).debug_sql:
- if (env.parent or env).config['trac'].get('debug_sql', False):
+ if trac.db.api.DatabaseManager(env).debug_sql:
log = env.log
product_prefix = env.product.prefix if env.product else GLOBAL_PRODUCT
translator = BloodhoundProductSQLTranslate(SKIP_TABLES,
TRANSLATE_TABLES,
PRODUCT_COLUMN,
- product_prefix)
+ product_prefix,
+ env)
if log:
log.debug('Original SQl: %s', sql)
realsql = translator.translate(sql) if (translator is not None) else sql
if log:
log.debug('SQL: %s', realsql)
+
return realsql
class BloodhoundIterableCursor(trac.db.util.IterableCursor):
@@ -202,11 +204,12 @@ class BloodhoundProductSQLTranslate(obje
'JOIN', 'INNER JOIN']
_from_end_words = ['WHERE', 'GROUP', 'HAVING', 'ORDER', 'UNION', 'LIMIT']
- def __init__(self, skip_tables, translate_tables, product_column,
product_prefix):
+ def __init__(self, skip_tables, translate_tables, product_column,
product_prefix, env=None):
self._skip_tables = skip_tables
self._translate_tables = translate_tables
self._product_column = product_column
self._product_prefix = product_prefix
+ self._using_sqlite = env is None or using_sqlite_backend(env)
def _sqlparse_underline_hack(self, token):
underline_token = lambda token: token.ttype == Tokens.Token.Error and
token.value == '_'
@@ -524,7 +527,8 @@ class BloodhoundProductSQLTranslate(obje
token = self._token_next(parent, start_token)
if not token.match(Tokens.Keyword, 'INTO'):
raise Exception("Invalid INSERT statement")
- def insert_extra_column(tablename, columns_token):
+ def insert_extra_columns(tablename, columns_token):
+ columns_present = []
if tablename in self._translate_tables and \
isinstance(columns_token, Types.Parenthesis):
ptoken = self._token_first(columns_token)
@@ -534,26 +538,56 @@ class BloodhoundProductSQLTranslate(obje
last_token = ptoken
while ptoken:
if isinstance(ptoken, Types.IdentifierList):
- if any(i.get_name() == 'product'
- for i in ptoken.get_identifiers()
- if isinstance(i, Types.Identifier)):
- return True
+ if not 'product' in columns_present \
+ and any(i.get_name() == 'product'
+ for i in ptoken.get_identifiers()
+ if isinstance(i, Types.Identifier)):
+ columns_present.append('product')
+ elif not 'id' in columns_present \
+ and tablename == 'ticket' \
+ and isinstance(ptoken, Types.IdentifierList) \
+ and any((t.ttype is None or t.is_keyword)
+ and t.value == 'id'
+ for t in ptoken.get_identifiers()):
+ columns_present.append('id')
last_token = ptoken
ptoken = self._token_next(columns_token, ptoken)
if not last_token or \
not last_token.match(Tokens.Punctuation, ')'):
raise Exception("Invalid INSERT statement, unable to find
column parenthesis end")
- for keyword in [',', ' ', self._product_column]:
+
+ columns_to_insert = []
+ if not 'product' in columns_present:
+ columns_to_insert += [',', ' ', self._product_column]
+ if self._using_sqlite \
+ and tablename == 'ticket'\
+ and not 'id' in columns_present:
+ columns_to_insert += [',', ' ', 'id']
+ for keyword in columns_to_insert:
self._token_insert_before(columns_token, last_token,
Types.Token(Tokens.Keyword, keyword))
- return False
- def insert_extra_column_value(tablename, ptoken, before_token):
+ return columns_present
+
+ def insert_extra_column_values(tablename, ptoken, before_token,
+ columns_present):
if tablename in self._translate_tables:
- for keyword in [',', "'", self._product_prefix, "'"]:
+ values_to_insert = []
+ if not 'product' in columns_present:
+ values_to_insert += [',', "'", self._product_prefix, "'"]
+ if self._using_sqlite \
+ and tablename == 'ticket' \
+ and not 'id' in columns_present:
+ values_to_insert += [
+ ',', """COALESCE((SELECT MAX(id) FROM ticket
+ WHERE product='%s'), 0)+1""" %
+ (self._product_prefix,)
+ ]
+ for keyword in values_to_insert:
self._token_insert_before(ptoken, before_token,
Types.Token(Tokens.Keyword, keyword))
return
+
tablename = None
table_name_token = self._token_next(parent, token)
- has_product_column = False
+ columns_present = []
if isinstance(table_name_token, Types.Function):
token = self._token_first(table_name_token)
if isinstance(token, Types.Identifier):
@@ -562,7 +596,7 @@ class BloodhoundProductSQLTranslate(obje
if columns_token.match(Tokens.Keyword, 'VALUES'):
token = columns_token
else:
- has_product_column = insert_extra_column(tablename,
columns_token)
+ columns_present = insert_extra_columns(tablename,
columns_token)
token = self._token_next(parent, table_name_token)
else:
tablename = table_name_token.value
@@ -570,11 +604,10 @@ class BloodhoundProductSQLTranslate(obje
if columns_token.match(Tokens.Keyword, 'VALUES'):
token = columns_token
else:
- has_product_column = insert_extra_column(tablename,
columns_token)
+ columns_present = insert_extra_columns(tablename,
columns_token)
token = self._token_next(parent, columns_token)
- if has_product_column:
- pass # INSERT already has product, no translation needed
- elif token.match(Tokens.Keyword, 'VALUES'):
+
+ if token.match(Tokens.Keyword, 'VALUES'):
separators = [',', '(', ')']
token = self._token_next(parent, token)
while token:
@@ -594,7 +627,7 @@ class BloodhoundProductSQLTranslate(obje
if not last_token or \
not last_token.match(Tokens.Punctuation, ')'):
raise Exception("Invalid INSERT statement, unable to
find column value parenthesis end")
- insert_extra_column_value(tablename, token, last_token)
+ insert_extra_column_values(tablename, token, last_token,
columns_present)
elif not token.match(Tokens.Punctuation, separators) and\
not token.match(Tokens.Keyword, separators) and\
not token.is_whitespace():
Modified:
bloodhound/branches/bep_0010_ticket_numbering/bloodhound_multiproduct/multiproduct/util.py
URL:
http://svn.apache.org/viewvc/bloodhound/branches/bep_0010_ticket_numbering/bloodhound_multiproduct/multiproduct/util.py?rev=1506493&r1=1506492&r2=1506493&view=diff
==============================================================================
---
bloodhound/branches/bep_0010_ticket_numbering/bloodhound_multiproduct/multiproduct/util.py
(original)
+++
bloodhound/branches/bep_0010_ticket_numbering/bloodhound_multiproduct/multiproduct/util.py
Wed Jul 24 10:56:43 2013
@@ -20,6 +20,7 @@
from genshi.builder import tag
+import trac.db.api
from trac.util.text import unquote_label
from trac.wiki.formatter import LinkFormatter
from trac.core import Component, ComponentMeta
@@ -139,6 +140,13 @@ class EmbeddedLinkFormatter(LinkFormatte
else fullmatch.group('label'))
return self._make_lhref_link(match, fullmatch, rel, ns, target, label)
+
+def using_sqlite_backend(env):
+ connector, arg = trac.db.api._parse_db_str(
+ trac.db.api.DatabaseManager(env).connection_uri
+ )
+ return connector == 'sqlite'
+
#----------------------
# Useful regex
#----------------------
Modified:
bloodhound/branches/bep_0010_ticket_numbering/bloodhound_multiproduct/tests/db/cursor.py
URL:
http://svn.apache.org/viewvc/bloodhound/branches/bep_0010_ticket_numbering/bloodhound_multiproduct/tests/db/cursor.py?rev=1506493&r1=1506492&r2=1506493&view=diff
==============================================================================
---
bloodhound/branches/bep_0010_ticket_numbering/bloodhound_multiproduct/tests/db/cursor.py
(original)
+++
bloodhound/branches/bep_0010_ticket_numbering/bloodhound_multiproduct/tests/db/cursor.py
Wed Jul 24 10:56:43 2013
@@ -992,7 +992,7 @@ data = {
'insert_with_product': [
(
"""INSERT INTO ticket (summary, product) VALUES ('S', 'swlcu')""",
-"""INSERT INTO ticket (summary, product) VALUES ('S', 'swlcu')"""
+"""INSERT INTO ticket (summary, product, id) VALUES ('S',
'swlcu',COALESCE((SELECT MAX(id) FROM ticket\nWHERE product='PRODUCT'), 0)+1)"""
),
],
Modified:
bloodhound/branches/bep_0010_ticket_numbering/bloodhound_multiproduct/tests/ticket/model.py
URL:
http://svn.apache.org/viewvc/bloodhound/branches/bep_0010_ticket_numbering/bloodhound_multiproduct/tests/ticket/model.py?rev=1506493&r1=1506492&r2=1506493&view=diff
==============================================================================
---
bloodhound/branches/bep_0010_ticket_numbering/bloodhound_multiproduct/tests/ticket/model.py
(original)
+++
bloodhound/branches/bep_0010_ticket_numbering/bloodhound_multiproduct/tests/ticket/model.py
Wed Jul 24 10:56:43 2013
@@ -22,12 +22,13 @@ from datetime import datetime
import shutil
import unittest
-from trac.ticket.model import Milestone
+from trac.ticket.model import Milestone, Ticket
from trac.ticket.tests.model import TicketTestCase, TicketCommentTestCase, \
TicketCommentEditTestCase, TicketCommentDeleteTestCase, EnumTestCase, \
MilestoneTestCase, ComponentTestCase, VersionTestCase
from trac.util.datefmt import to_utimestamp, utc
+from multiproduct.model import Product
from multiproduct.env import ProductEnvironment
from tests.env import MultiproductTestCase
@@ -47,6 +48,46 @@ class ProductTicketTestCase(TicketTestCa
self.global_env.reset_db()
self.env = self.global_env = None
+ def _get_ticket_uid(self, tid):
+ with self.env.db_query as db:
+ rows = db("""SELECT uid FROM ticket WHERE id=%s""", (tid, ))
+ return rows[0][0] if rows else -1
+
+ def test_insert_into_multiple_products(self):
+ # UIDs are global, autoincremented
+ # IDs are product-scoped, incremented in the SQL translator
+ self.env = ProductEnvironment(self.global_env, self.default_product)
+
+ tid = self._insert_ticket('hello kitty', reporter='admin')
+ ticket = Ticket(self.env, tid)
+ self.assertEqual(tid, 1)
+ self.assertEqual(self._get_ticket_uid(tid), 1)
+ self.assertEqual(ticket.id, tid)
+ tid = self._insert_ticket('hello kitteh', reporter='admin')
+ ticket = Ticket(self.env, tid)
+ self.assertEqual(tid, 2)
+ self.assertEqual(self._get_ticket_uid(tid), 2)
+ self.assertEqual(ticket.id, tid)
+
+ p2 = Product(self.global_env)
+ p2.prefix = 'p2'
+ p2.name = 'product, too'
+ p2.owner = 'admin'
+ p2.insert()
+ self.env = ProductEnvironment(self.global_env, p2)
+
+ tid = self._insert_ticket('hello catty', reporter='admin')
+ ticket = Ticket(self.env, tid)
+ self.assertEqual(tid, 1)
+ self.assertEqual(self._get_ticket_uid(tid), 3)
+ self.assertEqual(ticket.id, tid)
+ tid = self._insert_ticket('hello ocelot', reporter='admin')
+ ticket = Ticket(self.env, tid)
+ self.assertEqual(tid, 2)
+ self.assertEqual(self._get_ticket_uid(tid), 4)
+ self.assertEqual(ticket.id, tid)
+
+
class ProductTicketCommentTestCase(MultiproductTestCase):
@property
Modified:
bloodhound/branches/bep_0010_ticket_numbering/bloodhound_multiproduct/tests/upgrade.py
URL:
http://svn.apache.org/viewvc/bloodhound/branches/bep_0010_ticket_numbering/bloodhound_multiproduct/tests/upgrade.py?rev=1506493&r1=1506492&r2=1506493&view=diff
==============================================================================
---
bloodhound/branches/bep_0010_ticket_numbering/bloodhound_multiproduct/tests/upgrade.py
(original)
+++
bloodhound/branches/bep_0010_ticket_numbering/bloodhound_multiproduct/tests/upgrade.py
Wed Jul 24 10:56:43 2013
@@ -351,6 +351,66 @@ class EnvironmentUpgradeTestCase(unittes
self.assertEqual(len(products), 1)
self.assertEqual(products[0].prefix, 'xxx')
+ def test_migration_to_multiproduct_preserves_ticket_ids(self):
+ for ticket_id in (1, 3, 5, 7):
+ with self.env.db_transaction as db:
+ cursor = db.cursor()
+ cursor.execute("INSERT INTO ticket (id) VALUES (%i)" %
ticket_id)
+ db.update_sequence(cursor, 'ticket')
+
+ self._enable_multiproduct()
+ self.env.upgrade()
+
+ for ticket_id in (1, 3, 5, 7):
+ with self.product('@'):
+ ticket = Ticket(self.env, ticket_id)
+ self.assertEqual(ticket.id, ticket_id)
+
+ def test_can_insert_tickets_after_upgrade(self):
+ t1 = Ticket(self.env)
+ t1.summary = "test"
+ t1.insert()
+ self.assertEqual(t1.id, 1)
+
+ self._enable_multiproduct()
+ self.env.upgrade()
+
+ with self.product('@'):
+ ticket = Ticket(self.env)
+ ticket.summary = 'test'
+ ticket.insert()
+ self.assertEqual(ticket.id, 2)
+
+ def test_can_insert_tickets_with_same_id_to_different_products(self):
+ self._enable_multiproduct()
+ self.env.upgrade()
+
+ self.env.db_transaction("INSERT INTO ticket (id, summary)"
+ " VALUES (1, 'first product')")
+ t1 = Ticket(self.env, 1)
+
+ with self.product('@'):
+ self.env.db_transaction("INSERT INTO ticket (id, summary)"
+ " VALUES (1, 'second product')")
+ t2 = Ticket(self.env, 1)
+
+ self.assertEqual(t1.id, t2.id)
+ self.assertNotEqual(t1['summary'], t2['summary'])
+
+ def test_batch_ticket_insert_after_upgrade(self):
+ self._enable_multiproduct()
+ self.env.upgrade()
+ with self.env.db_direct_transaction as db:
+ db("""CREATE TABLE "@_tmp" (summary text, product text)""")
+ for summary in "abcdef":
+ db("""INSERT INTO "@_tmp" VALUES ('%s', '@')""" % (summary,))
+
+ with self.product('@'):
+ with self.env.db_transaction as db:
+ db("""INSERT INTO ticket (summary) SELECT summary FROM tmp""")
+
+
+
def _enable_multiproduct(self):
self._update_config('components', 'multiproduct.*', 'enabled')
Modified:
bloodhound/branches/bep_0010_ticket_numbering/trac/trac/ticket/model.py
URL:
http://svn.apache.org/viewvc/bloodhound/branches/bep_0010_ticket_numbering/trac/trac/ticket/model.py?rev=1506493&r1=1506492&r2=1506493&view=diff
==============================================================================
--- bloodhound/branches/bep_0010_ticket_numbering/trac/trac/ticket/model.py
(original)
+++ bloodhound/branches/bep_0010_ticket_numbering/trac/trac/ticket/model.py Wed
Jul 24 10:56:43 2013
@@ -226,7 +226,6 @@ class Ticket(object):
# Insert ticket record
std_fields = []
custom_fields = []
- print self.values
for f in self.fields:
fname = f['name']
if fname in self.values:
@@ -240,7 +239,13 @@ class Ticket(object):
% (','.join(std_fields),
','.join(['%s'] * len(std_fields))),
[values[name] for name in std_fields])
- tkt_id = db.get_last_id(cursor, 'ticket')
+ if getattr(self.env, '_multiproduct_schema_enabled', False):
+ tkt_id = db.get_last_id(cursor, 'ticket', 'uid')
+ rows = db("""SELECT id FROM ticket WHERE uid=%s""", (tkt_id,))
+ tkt_id = rows[0][0] if rows else -1
+ else:
+ tkt_id = db.get_last_id(cursor, 'ticket')
+
# Insert custom fields
if custom_fields:
