KUDU-1611 - [python] Enable setting scanner selection policy Currently the python client cannot set the scanner selection policy. This patch enables that both for the Scanner class and the ScanTokenBuilder. This patch includes two unit tests.
Change-Id: I09d9cd081c037e3284370943e2cae348476df26c Reviewed-on: http://gerrit.cloudera.org:8080/4408 Tested-by: Kudu Jenkins Reviewed-by: David Ribeiro Alves <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/d38a17d9 Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/d38a17d9 Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/d38a17d9 Branch: refs/heads/master Commit: d38a17d9be48fbe2ad3fc5aa9c034157b6b0a456 Parents: b7497b3 Author: Jordan Birdsell <[email protected]> Authored: Tue Sep 13 23:33:47 2016 -0400 Committer: David Ribeiro Alves <[email protected]> Committed: Fri Oct 28 17:01:50 2016 +0000 ---------------------------------------------------------------------- python/kudu/__init__.py | 3 ++ python/kudu/client.pyx | 81 ++++++++++++++++++++++++++++++++ python/kudu/libkudu_client.pxd | 6 +-- python/kudu/tests/test_scanner.py | 16 +++++++ python/kudu/tests/test_scantoken.py | 23 +++++++++ 5 files changed, 126 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kudu/blob/d38a17d9/python/kudu/__init__.py ---------------------------------------------------------------------- diff --git a/python/kudu/__init__.py b/python/kudu/__init__.py index 771f99a..8ff299c 100644 --- a/python/kudu/__init__.py +++ b/python/kudu/__init__.py @@ -19,6 +19,9 @@ from kudu.client import (Client, Table, Scanner, Session, # noqa Insert, Update, Delete, Predicate, TimeDelta, KuduError, ScanTokenBuilder, ScanToken, + LEADER_ONLY, + CLOSEST_REPLICA, + FIRST_REPLICA, FLUSH_AUTO_BACKGROUND, FLUSH_AUTO_SYNC, FLUSH_MANUAL, http://git-wip-us.apache.org/repos/asf/kudu/blob/d38a17d9/python/kudu/client.pyx ---------------------------------------------------------------------- diff --git a/python/kudu/client.pyx b/python/kudu/client.pyx index 97ae711..19ab7a8 100644 --- a/python/kudu/client.pyx +++ b/python/kudu/client.pyx @@ -34,6 +34,16 @@ from errors import KuduException import six +# Replica selection enums +LEADER_ONLY = ReplicaSelection_Leader +CLOSEST_REPLICA = ReplicaSelection_Closest +FIRST_REPLICA = ReplicaSelection_First + +cdef dict _replica_selection_policies = { + 'leader': ReplicaSelection_Leader, + 'closest': ReplicaSelection_Closest, + 'first': ReplicaSelection_First +} # Read mode enums READ_LATEST = ReadMode_Latest @@ -1336,6 +1346,41 @@ cdef class Scanner: check_status(self.scanner.SetProjectedColumnNames(v_names)) return self + def set_selection(self, replica_selection): + """ + Set the replica selection policy while scanning. + + Parameters + ---------- + replica_selection : {'leader', 'closest', 'first'} + You can also use the constants LEADER_ONLY, CLOSEST_REPLICA, + and FIRST_REPLICA + + Returns + ------- + self : Scanner + """ + cdef ReplicaSelection selection + + def invalid_selection_policy(): + raise ValueError('Invalid replica selection policy: {0}' + .format(replica_selection)) + + if isinstance(replica_selection, int): + if 0 <= replica_selection < len(_replica_selection_policies): + check_status(self.scanner.SetSelection( + <ReplicaSelection> replica_selection)) + else: + invalid_selection_policy() + else: + try: + check_status(self.scanner.SetSelection( + _replica_selection_policies[replica_selection.lower()])) + except KeyError: + invalid_selection_policy() + + return self + def set_projected_column_indexes(self, indexes): """ Sets the columns to be scanned. @@ -1578,6 +1623,7 @@ cdef class Scanner: check_status(self.scanner.NextBatch(&batch.batch)) return batch + cdef class ScanToken: """ A ScanToken describes a partial scan of a Kudu table limited to a single @@ -1826,6 +1872,41 @@ cdef class ScanTokenBuilder: check_status(self._builder.SetFaultTolerant()) return self + def set_selection(self, replica_selection): + """ + Set the replica selection policy while scanning. + + Parameters + ---------- + replica_selection : {'leader', 'closest', 'first'} + You can also use the constants LEADER_ONLY, CLOSEST_REPLICA, + and FIRST_REPLICA + + Returns + ------- + self : ScanTokenBuilder + """ + cdef ReplicaSelection selection + + def invalid_selection_policy(): + raise ValueError('Invalid replica selection policy: {0}' + .format(replica_selection)) + + if isinstance(replica_selection, int): + if 0 <= replica_selection < len(_replica_selection_policies): + check_status(self._builder.SetSelection( + <ReplicaSelection> replica_selection)) + else: + invalid_selection_policy() + else: + try: + check_status(self._builder.SetSelection( + _replica_selection_policies[replica_selection.lower()])) + except KeyError: + invalid_selection_policy() + + return self + def add_predicates(self, preds): """ Add a list of scan predicates to the ScanTokenBuilder. Select columns http://git-wip-us.apache.org/repos/asf/kudu/blob/d38a17d9/python/kudu/libkudu_client.pxd ---------------------------------------------------------------------- diff --git a/python/kudu/libkudu_client.pxd b/python/kudu/libkudu_client.pxd index 83a9b03..141fdb7 100644 --- a/python/kudu/libkudu_client.pxd +++ b/python/kudu/libkudu_client.pxd @@ -453,9 +453,9 @@ cdef extern from "kudu/client/value.h" namespace "kudu::client" nogil: cdef extern from "kudu/client/client.h" namespace "kudu::client" nogil: enum ReplicaSelection" kudu::client::KuduClient::ReplicaSelection": - LEADER_ONLY " kudu::client::KuduClient::LEADER_ONLY" - CLOSEST_REPLICA " kudu::client::KuduClient::CLOSEST_REPLICA" - FIRST_REPLICA " kudu::client::KuduClient::FIRST_REPLICA" + ReplicaSelection_Leader " kudu::client::KuduClient::LEADER_ONLY" + ReplicaSelection_Closest " kudu::client::KuduClient::CLOSEST_REPLICA" + ReplicaSelection_First " kudu::client::KuduClient::FIRST_REPLICA" enum ReadMode" kudu::client::KuduScanner::ReadMode": ReadMode_Latest " kudu::client::KuduScanner::READ_LATEST" http://git-wip-us.apache.org/repos/asf/kudu/blob/d38a17d9/python/kudu/tests/test_scanner.py ---------------------------------------------------------------------- diff --git a/python/kudu/tests/test_scanner.py b/python/kudu/tests/test_scanner.py index ebacb72..fd39f1a 100644 --- a/python/kudu/tests/test_scanner.py +++ b/python/kudu/tests/test_scanner.py @@ -258,3 +258,19 @@ class TestScanner(TestScanBase): # Test a single precision float predicate # Does a row check count only self._test_float_pred() + + def test_scan_selection(self): + """ + This test confirms that setting the scan selection policy on the + scanner does not cause any errors. There is no way to confirm + that the policy was actually set. This functionality is + tested in the C++ test: + ClientTest.TestReplicatedMultiTabletTableFailover. + """ + + for policy in ['leader', kudu.CLOSEST_REPLICA, 2]: + scanner = self.table.scanner() + scanner.set_selection(policy) + scanner.open() + self.assertEqual(sorted(scanner.read_all_tuples()), + sorted(self.tuples)) http://git-wip-us.apache.org/repos/asf/kudu/blob/d38a17d9/python/kudu/tests/test_scantoken.py ---------------------------------------------------------------------- diff --git a/python/kudu/tests/test_scantoken.py b/python/kudu/tests/test_scantoken.py index 897d780..fbe66df 100644 --- a/python/kudu/tests/test_scantoken.py +++ b/python/kudu/tests/test_scantoken.py @@ -239,3 +239,26 @@ class TestScanToken(TestScanBase): # Test a single precision float predicate # Does a row check count only self._test_float_pred() + + def test_scan_selection(self): + """ + This test confirms that setting the scan selection policy on the + ScanTokenBuilder does not cause any errors . There is no way to + confirm that the policy was actually set. This functionality is + tested in the C++ test: + ClientTest.TestReplicatedMultiTabletTableFailover. + """ + + for policy in ['leader', kudu.CLOSEST_REPLICA, 2]: + builder = self.table.scan_token_builder() + builder.set_selection(policy) + tokens = builder.build() + + tuples = [] + for token in tokens: + scanner = token.into_kudu_scanner() + scanner.open() + tuples.extend(scanner.read_all_tuples()) + + self.assertEqual(sorted(tuples), + sorted(self.tuples))
