Ottomata has submitted this change and it was merged. (
https://gerrit.wikimedia.org/r/365999 )
Change subject: Auto add indexes for date-time and uuid v1 type columns
......................................................................
Auto add indexes for date-time and uuid v1 type columns
Bug: T170925
Change-Id: I8667916bf6f99da5aa4e9422747b113881955f8b
---
M eventlogging/jrm.py
M tests/test_jrm.py
2 files changed, 53 insertions(+), 3 deletions(-)
Approvals:
Mforns: Looks good to me, approved
Ottomata: Verified; Looks good to me, approved
diff --git a/eventlogging/jrm.py b/eventlogging/jrm.py
index 1b57b96..3fe3ef5 100644
--- a/eventlogging/jrm.py
+++ b/eventlogging/jrm.py
@@ -11,6 +11,7 @@
import collections
import datetime
+import dateutil.parser
import itertools
import logging
import _mysql
@@ -55,7 +56,6 @@
return '{}_{}'.format(table_name, event.schema_revision())
else:
return table_name
-
# An iterable of properties that should not be stored in the database.
@@ -105,8 +105,28 @@
return datetime.datetime.strptime(value, MEDIAWIKI_TIMESTAMP)
+class DateTimeSerde(sqlalchemy.TypeDecorator):
+ """
+ A :class:`sqlalchemy.TypeDecorator` for datetime columns.
+ This simply uses dateutil.parser.parse on the incoming value
+ to return a Python datetime, which is then given to the
+ sqlalchemy.DateTime column type. Conversion from
+ sqlalchemy.DateTime back to python datetime is handled
+ by sqlalchemy, so process_result_value is not implemented.
+ """
+
+ impl = sqlalchemy.DateTime
+
+ def process_bind_param(self, value, dialect=None):
+ """Converts a time string to a python datetime."""
+ return dateutil.parser.parse(value)
+
+
class JsonSerde(sqlalchemy.TypeDecorator):
- """A :class:`sqlalchemy.TypeDecorator` for converting to and from JSON
strings."""
+ """
+ A :class:`sqlalchemy.TypeDecorator` for converting
+ to and from JSON strings.
+ """
impl = sqlalchemy.Unicode(STRING_MAX_LEN)
@@ -145,6 +165,10 @@
'utc-millisec': {'type_': MediaWikiTimestamp, 'index': True},
'uuid5-hex': {'type_': sqlalchemy.CHAR(32), 'index': True,
'unique': True},
+ # Add indexes to datetime fields: T170925
+ # NOTE: I tried to use timestamp instead of datetime, but could
+ # not get sqlalchemy to create those as DEFAULT NULL.
+ 'date-time': {'type_': DateTimeSerde, 'index': True}
}),
('required', {
# Note that required=true makes the column nullable anyway.
@@ -155,6 +179,14 @@
# required field is present, or discard the event otherwise.
True: {'nullable': True},
False: {'nullable': True}
+ }),
+ ('pattern', {
+ # UUID v1 pattern, make a unique index. T170925
+ '^[a-fA-F0-9]{8}(-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}$': {
+ 'type_': sqlalchemy.CHAR(38),
+ 'index': True,
+ 'unique': True
+ },
})
))
diff --git a/tests/test_jrm.py b/tests/test_jrm.py
index 7559ccf..a231a00 100644
--- a/tests/test_jrm.py
+++ b/tests/test_jrm.py
@@ -68,13 +68,31 @@
cols |= {'id', 'uuid'} # plus id, uuid
self.assertSetEqual(set(t.columns.keys()), cols)
- def test_index_creation(self):
+ def test_timestamp_column_index_creation(self):
"""The ``timestamp`` column is indexed by default."""
table_name = event_to_table_name(self.event)
t = eventlogging.jrm.declare_table(self.meta, TEST_SCHEMA_SCID,
table_name)
cols = {column.name for index in t.indexes for column in index.columns}
self.assertIn('timestamp', cols)
+ def test_date_time_type_index_creation(self):
+ """A ``date-time`` column is indexed by default."""
+ table_name = event_to_table_name(self.event_with_meta)
+ t = eventlogging.jrm.declare_table(
+ self.meta, TEST_META_SCHEMA_SCID, table_name,
should_encapsulate=False
+ )
+ cols = {column.name for index in t.indexes for column in index.columns}
+ self.assertIn('meta_dt', cols)
+
+ def test_uuid_v1_type_index_creation(self):
+ """A ``uuid v1 pattern`` column is indexed by default."""
+ table_name = event_to_table_name(self.event_with_meta)
+ t = eventlogging.jrm.declare_table(
+ self.meta, TEST_META_SCHEMA_SCID, table_name,
should_encapsulate=False
+ )
+ cols = {column.name for index in t.indexes for column in index.columns}
+ self.assertIn('meta_id', cols)
+
def test_flatten(self):
"""``flatten`` correctly collapses deeply nested maps."""
flat = eventlogging.utils.flatten(self.event)
--
To view, visit https://gerrit.wikimedia.org/r/365999
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I8667916bf6f99da5aa4e9422747b113881955f8b
Gerrit-PatchSet: 2
Gerrit-Project: eventlogging
Gerrit-Branch: master
Gerrit-Owner: Ottomata <[email protected]>
Gerrit-Reviewer: Kaldari <[email protected]>
Gerrit-Reviewer: Mforns <[email protected]>
Gerrit-Reviewer: Nuria <[email protected]>
Gerrit-Reviewer: Ottomata <[email protected]>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
