[MediaWiki-commits] [Gerrit] eventlogging[master]: Test batching of events for sql insertion
Mforns has submitted this change and it was merged. (
https://gerrit.wikimedia.org/r/363981 )
Change subject: Test batching of events for sql insertion
..
Test batching of events for sql insertion
Events are batched given a schema id, schema version
and topic. Events that share schemas but not topics should be
in different batches.
Bug: T150369
Change-Id: I5feaeb297a690f0b1896ea11c31de1e47096caca
---
M requirements.txt
M tests/test_handlers.py
2 files changed, 59 insertions(+), 0 deletions(-)
Approvals:
Mforns: Verified; Looks good to me, approved
diff --git a/requirements.txt b/requirements.txt
index fd3d7b3..b22e00f 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -2,6 +2,7 @@
jsonschema>=0.7
confluent-kafka>=0.9.1.2
kafka-python>=1.3.0
+mock==1.0.0
mysqlclient>=1.3.7
pygments>=1.5
psutil>=2.1.1
diff --git a/tests/test_handlers.py b/tests/test_handlers.py
index a6e4895..388c9e8 100644
--- a/tests/test_handlers.py
+++ b/tests/test_handlers.py
@@ -10,11 +10,13 @@
import os
import unittest
+from mock import patch, Mock, MagicMock
import eventlogging
import eventlogging.handlers
import eventlogging.factory
+from .fixtures import _get_event
def echo_writer(uri, **kwargs):
values = []
@@ -51,6 +53,62 @@
reader = eventlogging.get_reader('test://localhost/?value=secret')
self.assertEqual(next(reader), 'secret')
+class SQLHandlerTestCase(unittest.TestCase):
+
+def test_sql_batching_happy_case_same_schema(self):
+"""
+Send several events that will get batched together
+as they belong to the same schema
+"""
+
+# Patching works in the scope of the function that calls
+# it, until python3 doesn't play seameslly with unittest
+@patch('eventlogging.handlers.store_sql_events')
+def mock_holder(mock_store_sql_events):
+
+writer =
eventlogging.get_writer('sqlite://?batch_size=3_time=10')
+event = _get_event().next()
+
+writer.send(event)
+writer.send(event)
+self.assertEqual(mock_store_sql_events.call_count, 0, 'No call to
insert should happened, batch size not reached')
+# the two events belong to the same batch the store_sql_events
should have
+writer.send(event)
+self.assertEqual(mock_store_sql_events.call_count, 1, 'Reached
batch size, should have inserted')
+writer.send(event)
+self.assertEqual(mock_store_sql_events.call_count, 1, 'No call to
insert should happened, batch size not reached')
+
+mock_holder();
+
+def test_sql_batching_schemas_and_topics(self):
+"""
+Send several events that will get batched separately
+as they belong to different topics but same schema
+"""
+# Patching works in the scope of the function that calls
+# it, until python3 doesn't play seamlessly with unittest
+@patch('eventlogging.handlers.store_sql_events')
+def mock_holder(mock_store_sql_events):
+writer =
eventlogging.get_writer('sqlite://?batch_size=3_time=10')
+event_topic1 = _get_event().next()
+event_topic2 = _get_event().next()
+
+event_topic2['topic'] = 'different_test_topic'
+writer.send(event_topic1)
+writer.send(event_topic1)
+writer.send(event_topic2)
+writer.send(event_topic2)
+
+self.assertEqual(mock_store_sql_events.call_count, 0, 'No call to
insert should happened, batch size not reached')
+# now add a new event on second topic
+writer.send(event_topic2)
+self.assertEqual(mock_store_sql_events.call_count, 1, 'Reached
batch size, should have inserted')
+# add a new topic1 event
+writer.send(event_topic1)
+self.assertEqual(mock_store_sql_events.call_count, 2, 'Reached
batch size, should have inserted')
+
+mock_holder();
+
class PluginTestCase(unittest.TestCase):
"""Test case for the plug-in loader."""
--
To view, visit https://gerrit.wikimedia.org/r/363981
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I5feaeb297a690f0b1896ea11c31de1e47096caca
Gerrit-PatchSet: 4
Gerrit-Project: eventlogging
Gerrit-Branch: master
Gerrit-Owner: Nuria
Gerrit-Reviewer: Kaldari
Gerrit-Reviewer: Mforns
Gerrit-Reviewer: Nuria
Gerrit-Reviewer: Ottomata
___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
[MediaWiki-commits] [Gerrit] eventlogging[master]: Test batching of events for sql insertion
Nuria has uploaded a new change for review. (
https://gerrit.wikimedia.org/r/363981 )
Change subject: Test batching of events for sql insertion
..
Test batching of events for sql insertion
Events are batched given a schema id, version and topic
Events that share schemas but not topics should be
in different batches
Bug: T150369
Change-Id: I5feaeb297a690f0b1896ea11c31de1e47096caca
---
M requirements.txt
M tests/test_handlers.py
2 files changed, 57 insertions(+), 1 deletion(-)
git pull ssh://gerrit.wikimedia.org:29418/eventlogging
refs/changes/81/363981/1
diff --git a/requirements.txt b/requirements.txt
index fd3d7b3..b22e00f 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -2,6 +2,7 @@
jsonschema>=0.7
confluent-kafka>=0.9.1.2
kafka-python>=1.3.0
+mock==1.0.0
mysqlclient>=1.3.7
pygments>=1.5
psutil>=2.1.1
diff --git a/tests/test_handlers.py b/tests/test_handlers.py
index a6e4895..add3f98 100644
--- a/tests/test_handlers.py
+++ b/tests/test_handlers.py
@@ -10,11 +10,13 @@
import os
import unittest
+from mock import patch, Mock, MagicMock
import eventlogging
import eventlogging.handlers
import eventlogging.factory
+from .fixtures import _get_event
def echo_writer(uri, **kwargs):
values = []
@@ -51,7 +53,60 @@
reader = eventlogging.get_reader('test://localhost/?value=secret')
self.assertEqual(next(reader), 'secret')
-
+class SQLHandlerTestCase(unittest.TestCase):
+
+def test_sql_batching_happy_case_same_schema(self):
+"""
+Send several events that will get batched together
+as they belong to the same schema
+"""
+
+# Patching works in teh scope of the function
+@patch('eventlogging.handlers.store_sql_events')
+def mock_holder(mock_store_sql_events):
+
+writer =
eventlogging.get_writer('sqlite://?batch_size=3_time=10')
+event = _get_event().next()
+
+writer.send(event)
+writer.send(event)
+self.assertEqual(mock_store_sql_events.call_count, 0, 'No call to
insert should happened, batch size not reached')
+# the two events belong to the same batch the store_sql_events
should have
+writer.send(event)
+self.assertEqual(mock_store_sql_events.call_count, 1, 'Reached
batch size, should have inserted')
+writer.send(event)
+self.assertEqual(mock_store_sql_events.call_count, 1, 'No call to
insert should happened, batch size not reached')
+
+mock_holder();
+
+def test_sql_batching_schemas_and_topics(self):
+"""
+Send several events that will get batched separately
+as they belong to different topics
+"""
+@patch('eventlogging.handlers.store_sql_events')
+def mock_holder(mock_store_sql_events):
+writer =
eventlogging.get_writer('sqlite://?batch_size=3_time=10')
+event_topic1 = _get_event().next()
+event_topic2 = _get_event().next()
+
+event_topic2['topic'] = 'different_test_topic'
+writer.send(event_topic1)
+writer.send(event_topic1)
+writer.send(event_topic2)
+writer.send(event_topic2)
+
+self.assertEqual(mock_store_sql_events.call_count, 0, 'No call to
insert should happened, batch size not reached')
+# now add a new event on second topic
+writer.send(event_topic2)
+self.assertEqual(mock_store_sql_events.call_count, 1, 'Reached
batch size, should have inserted')
+# add a new topic1 event
+writer.send(event_topic1)
+self.assertEqual(mock_store_sql_events.call_count, 2, 'Reached
batch size, should have inserted')
+
+mock_holder();
+
+
class PluginTestCase(unittest.TestCase):
"""Test case for the plug-in loader."""
--
To view, visit https://gerrit.wikimedia.org/r/363981
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I5feaeb297a690f0b1896ea11c31de1e47096caca
Gerrit-PatchSet: 1
Gerrit-Project: eventlogging
Gerrit-Branch: master
Gerrit-Owner: Nuria
___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
