Repository: kudu Updated Branches: refs/heads/master 84244461d -> 30cf90028
KUDU-1704: add python client support for READ_YOUR_WRITES mode This patch allows users to specify READ_YOUR_WRITES as a read mode in python client. It adds correponding tests to ensure the mode is actually working. Change-Id: I281a73ead2d606e698ff7f44ddb2cd1c78ffdd2a Reviewed-on: http://gerrit.cloudera.org:8080/9617 Reviewed-by: David Ribeiro Alves <[email protected]> Tested-by: Kudu Jenkins Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/9d233f45 Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/9d233f45 Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/9d233f45 Branch: refs/heads/master Commit: 9d233f457af6bea7f30779fe0b9c812859d90a02 Parents: 8424446 Author: hahao <[email protected]> Authored: Tue Mar 13 17:01:04 2018 -0700 Committer: Hao Hao <[email protected]> Committed: Wed Mar 14 20:48:36 2018 +0000 ---------------------------------------------------------------------- python/kudu/__init__.py | 1 + python/kudu/client.pyx | 9 ++++++--- python/kudu/libkudu_client.pxd | 1 + python/kudu/tests/test_scanner.py | 12 +++++++++--- python/kudu/tests/test_scantoken.py | 17 ++++++++++++++--- 5 files changed, 31 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kudu/blob/9d233f45/python/kudu/__init__.py ---------------------------------------------------------------------- diff --git a/python/kudu/__init__.py b/python/kudu/__init__.py index 75260d6..61c729d 100644 --- a/python/kudu/__init__.py +++ b/python/kudu/__init__.py @@ -27,6 +27,7 @@ from kudu.client import (Client, Table, Scanner, Session, # noqa FLUSH_MANUAL, READ_LATEST, READ_AT_SNAPSHOT, + READ_YOUR_WRITES, EXCLUSIVE_BOUND, INCLUSIVE_BOUND) http://git-wip-us.apache.org/repos/asf/kudu/blob/9d233f45/python/kudu/client.pyx ---------------------------------------------------------------------- diff --git a/python/kudu/client.pyx b/python/kudu/client.pyx index 34cb53a..ab43851 100644 --- a/python/kudu/client.pyx +++ b/python/kudu/client.pyx @@ -49,10 +49,12 @@ cdef dict _replica_selection_policies = { # Read mode enums READ_LATEST = ReadMode_Latest READ_AT_SNAPSHOT = ReadMode_Snapshot +READ_YOUR_WRITES = ReadMode_ReadYourWrites cdef dict _read_modes = { 'latest': ReadMode_Latest, - 'snapshot': ReadMode_Snapshot + 'snapshot': ReadMode_Snapshot, + 'read_your_writes': ReadMode_ReadYourWrites } cdef dict _type_names = { @@ -1557,8 +1559,9 @@ cdef class Scanner: Parameters ---------- - read_mode : {'latest', 'snapshot'} - You can also use the constants READ_LATEST, READ_AT_SNAPSHOT + read_mode : {'latest', 'snapshot', 'read_your_writes'} + You can also use the constants READ_LATEST, READ_AT_SNAPSHOT, + READ_YOUR_WRITES Returns ------- http://git-wip-us.apache.org/repos/asf/kudu/blob/9d233f45/python/kudu/libkudu_client.pxd ---------------------------------------------------------------------- diff --git a/python/kudu/libkudu_client.pxd b/python/kudu/libkudu_client.pxd index b834bf0..e2bc90d 100644 --- a/python/kudu/libkudu_client.pxd +++ b/python/kudu/libkudu_client.pxd @@ -496,6 +496,7 @@ cdef extern from "kudu/client/client.h" namespace "kudu::client" nogil: enum ReadMode" kudu::client::KuduScanner::ReadMode": ReadMode_Latest " kudu::client::KuduScanner::READ_LATEST" ReadMode_Snapshot " kudu::client::KuduScanner::READ_AT_SNAPSHOT" + ReadMode_ReadYourWrites " kudu::client::KuduScanner::READ_YOUR_WRITES" enum RangePartitionBound" kudu::client::KuduTableCreator::RangePartitionBound": PartitionType_Exclusive " kudu::client::KuduTableCreator::EXCLUSIVE_BOUND" http://git-wip-us.apache.org/repos/asf/kudu/blob/9d233f45/python/kudu/tests/test_scanner.py ---------------------------------------------------------------------- diff --git a/python/kudu/tests/test_scanner.py b/python/kudu/tests/test_scanner.py index fa94ea3..23d5462 100644 --- a/python/kudu/tests/test_scanner.py +++ b/python/kudu/tests/test_scanner.py @@ -182,8 +182,7 @@ class TestScanner(TestScanBase): def test_read_mode(self): """ - Test setting the read mode and scanning against a - snapshot and latest + Test scanning in latest, snapshot and read_your_writes read modes. """ # Delete row self.delete_insert_row_for_read_test() @@ -196,7 +195,7 @@ class TestScanner(TestScanBase): self.assertEqual(sorted(self.tuples[1:]), sorted(scanner.read_all_tuples())) - #Check scanner results after delete + # Check scanner results after delete with latest mode timeout = time.time() + 10 check_tuples = [] while check_tuples != sorted(self.tuples): @@ -211,6 +210,13 @@ class TestScanner(TestScanBase): # Avoid tight looping time.sleep(0.05) + # Check scanner results after delete with read_your_writes mode + scanner = self.table.scanner() + scanner.set_read_mode('read_your_writes')\ + .open() + + self.assertEqual(sorted(self.tuples), sorted(scanner.read_all_tuples())) + def test_resource_metrics_and_cache_blocks(self): """ Test getting the resource metrics after scanning and http://git-wip-us.apache.org/repos/asf/kudu/blob/9d233f45/python/kudu/tests/test_scantoken.py ---------------------------------------------------------------------- diff --git a/python/kudu/tests/test_scantoken.py b/python/kudu/tests/test_scantoken.py index c274668..7d664c0 100644 --- a/python/kudu/tests/test_scantoken.py +++ b/python/kudu/tests/test_scantoken.py @@ -193,8 +193,7 @@ class TestScanToken(TestScanBase): def test_read_mode(self): """ - Test setting the read mode and scanning against a - snapshot and latest + Test scanning in latest, snapshot and read_your_writes read modes. """ # Delete row self.delete_insert_row_for_read_test() @@ -212,7 +211,7 @@ class TestScanToken(TestScanBase): self.assertEqual(sorted(self.tuples[1:]), sorted(tuples)) - #Check scanner results after insterts + # Check scanner results after inserts with latest mode builder = self.table.scan_token_builder() tokens = builder.set_read_mode(kudu.READ_LATEST) \ .build() @@ -224,6 +223,18 @@ class TestScanToken(TestScanBase): self.assertEqual(sorted(self.tuples), sorted(tuples)) + # Check scanner results after inserts with read_your_writes mode + builder = self.table.scan_token_builder() + tokens = builder.set_read_mode(kudu.READ_YOUR_WRITES)\ + .build() + + tuples = [] + for token in tokens: + scanner = self._subtest_open_and_confirm_leader_tserver(token) + tuples.extend(scanner.read_all_tuples()) + + self.assertEqual(sorted(self.tuples), sorted(tuples)) + def verify_pred_type_scans(self, preds, row_indexes, count_only=False): # Using the incoming list of predicates, verify that the row returned # matches the inserted tuple at the row indexes specified in a
