kudu git commit: Remove namespace hack in hms_client.h

2018-03-07 Thread danburkert 
Repository: kudu
Updated Branches:
  refs/heads/master debcb8ea2 -> b525378c5


Remove namespace hack in hms_client.h

To avoid having to use the very long Apache.Hadoop.Hive namespace for
all of the Thrift-generated HMS types, we aliased the namespace in
hms_client.h to 'hive'. This doesn't really work well, I've seen issues
in tests where for unknown reasons the hive namespace doesn't work.

A better fix is to just edit hive_metastore.thrift to change the
generated namespace, since it's vendored anyway. This exposed that our
CMake thrift generator doesn't properly consider the input .thrift files
to be a dependency of the code generation step, so I had to fix that as
well to get this to compile without blowing away the build dir.

Change-Id: I1b4d823a0329c63c9ce830f855c3d1021fbc08cf
Reviewed-on: http://gerrit.cloudera.org:8080/9553
Reviewed-by: Adar Dembo 
Tested-by: Kudu Jenkins


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/b525378c
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/b525378c
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/b525378c

Branch: refs/heads/master
Commit: b525378c550a8227257ffac2a8d45a6d519f33f7
Parents: debcb8e
Author: Dan Burkert 
Authored: Wed Mar 7 19:03:21 2018 -0800
Committer: Dan Burkert 
Committed: Thu Mar 8 04:29:21 2018 +

--
 cmake_modules/FindThrift.cmake | 1 +
 src/kudu/hms/hive_metastore.thrift | 5 -
 src/kudu/hms/hms_client.h  | 2 --
 3 files changed, 5 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/b525378c/cmake_modules/FindThrift.cmake
--
diff --git a/cmake_modules/FindThrift.cmake b/cmake_modules/FindThrift.cmake
index 4016c20..834bbfd 100644
--- a/cmake_modules/FindThrift.cmake
+++ b/cmake_modules/FindThrift.cmake
@@ -108,6 +108,7 @@ function(THRIFT_GENERATE_CPP SRCS HDRS TGTS)
 
 add_custom_command(
   OUTPUT ${THRIFT_CC_OUT} ${THRIFT_H_OUT}
+  DEPENDS ${ABS_FIL}
   COMMAND  ${THRIFT_EXECUTABLE}
   ARGS
 --gen cpp:moveable_types

http://git-wip-us.apache.org/repos/asf/kudu/blob/b525378c/src/kudu/hms/hive_metastore.thrift
--
diff --git a/src/kudu/hms/hive_metastore.thrift 
b/src/kudu/hms/hive_metastore.thrift
index 448ce6c..969f4f8 100644
--- a/src/kudu/hms/hive_metastore.thrift
+++ b/src/kudu/hms/hive_metastore.thrift
@@ -23,6 +23,9 @@
 # With backports:
 #   - HIVE-16993
 #
+# With edits:
+#   - Change cpp namespace to 'hive' to match the Kudu codebase style.
+#
 # Before updating to a new version, consider that Kudu must remain compatible
 # with a range of Hive Metastore versions.
 
@@ -34,7 +37,7 @@ include "share/fb303/if/fb303.thrift"
 
 namespace java org.apache.hadoop.hive.metastore.api
 namespace php metastore
-namespace cpp Apache.Hadoop.Hive
+namespace cpp hive
 
 const string DDL_TIME = "transient_lastDdlTime"
 

http://git-wip-us.apache.org/repos/asf/kudu/blob/b525378c/src/kudu/hms/hms_client.h
--
diff --git a/src/kudu/hms/hms_client.h b/src/kudu/hms/hms_client.h
index a71fcf6..21f18f9 100644
--- a/src/kudu/hms/hms_client.h
+++ b/src/kudu/hms/hms_client.h
@@ -28,8 +28,6 @@
 #include "kudu/util/slice.h"
 #include "kudu/util/status.h"
 
-namespace hive = Apache::Hadoop::Hive;
-
 namespace kudu {
 
 class HostPort;

[1/2] kudu git commit: env: generalize resource limits and add RLIMIT_NPROC support

2018-03-07 Thread adar 
Repository: kudu
Updated Branches:
  refs/heads/master 954d9f71b -> debcb8ea2


env: generalize resource limits and add RLIMIT_NPROC support

A follow-on patch will use this to cap the max number of threads in some
process-wide thread pools (see KUDU-1913).

Change-Id: I567f581a6f8a85ac1f08878b61ac316cc2da36a0
Reviewed-on: http://gerrit.cloudera.org:8080/9521
Reviewed-by: David Ribeiro Alves 
Tested-by: Kudu Jenkins


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/ede0cf0e
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/ede0cf0e
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/ede0cf0e

Branch: refs/heads/master
Commit: ede0cf0ec3924a067f82b625eb61410e8b734183
Parents: 954d9f7
Author: Adar Dembo 
Authored: Tue Mar 6 15:45:40 2018 -0800
Committer: Adar Dembo 
Committed: Thu Mar 8 02:43:31 2018 +

--
 src/kudu/fs/block_manager.cc | 13 
 src/kudu/rpc/rpc-test.cc |  2 +-
 src/kudu/util/env-test.cc| 27 +
 src/kudu/util/env.h  | 35 +-
 src/kudu/util/env_posix.cc   | 62 +++
 5 files changed, 106 insertions(+), 33 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/ede0cf0e/src/kudu/fs/block_manager.cc
--
diff --git a/src/kudu/fs/block_manager.cc b/src/kudu/fs/block_manager.cc
index e5257c1..d3ce1c0 100644
--- a/src/kudu/fs/block_manager.cc
+++ b/src/kudu/fs/block_manager.cc
@@ -83,19 +83,20 @@ int64_t GetFileCacheCapacityForBlockManager(Env* env) {
   // Maximize this process' open file limit first, if possible.
   static std::once_flag once;
   std::call_once(once, [&]() {
-env->IncreaseOpenFileLimit();
+env->IncreaseResourceLimit(Env::ResourceLimitType::OPEN_FILES_PER_PROCESS);
   });
 
+  int64_t rlimit =
+  env->GetResourceLimit(Env::ResourceLimitType::OPEN_FILES_PER_PROCESS);
   // See block_manager_max_open_files.
   if (FLAGS_block_manager_max_open_files == -1) {
-return (2 * env->GetOpenFileLimit()) / 5;
+return (2 * rlimit) / 5;
   }
-  int64_t file_limit = env->GetOpenFileLimit();
-  LOG_IF(FATAL, FLAGS_block_manager_max_open_files > file_limit) <<
+  LOG_IF(FATAL, FLAGS_block_manager_max_open_files > rlimit) <<
   Substitute(
   "Configured open file limit (block_manager_max_open_files) $0 "
-  "exceeds process fd limit (ulimit) $1",
-  FLAGS_block_manager_max_open_files, file_limit);
+  "exceeds process open file limit (ulimit) $1",
+  FLAGS_block_manager_max_open_files, rlimit);
   return FLAGS_block_manager_max_open_files;
 }
 

http://git-wip-us.apache.org/repos/asf/kudu/blob/ede0cf0e/src/kudu/rpc/rpc-test.cc
--
diff --git a/src/kudu/rpc/rpc-test.cc b/src/kudu/rpc/rpc-test.cc
index f279bff..f6496cf 100644
--- a/src/kudu/rpc/rpc-test.cc
+++ b/src/kudu/rpc/rpc-test.cc
@@ -348,7 +348,7 @@ TEST_P(TestRpc, TestHighFDs) {
   // This test can only run if ulimit is set high.
   const int kNumFakeFiles = 3500;
   const int kMinUlimit = kNumFakeFiles + 100;
-  if (env_->GetOpenFileLimit() < kMinUlimit) {
+  if (env_->GetResourceLimit(Env::ResourceLimitType::OPEN_FILES_PER_PROCESS) < 
kMinUlimit) {
 LOG(INFO) << "Test skipped: must increase ulimit -n to at least " << 
kMinUlimit;
 return;
   }

http://git-wip-us.apache.org/repos/asf/kudu/blob/ede0cf0e/src/kudu/util/env-test.cc
--
diff --git a/src/kudu/util/env-test.cc b/src/kudu/util/env-test.cc
index c152b87..83bf67d 100644
--- a/src/kudu/util/env-test.cc
+++ b/src/kudu/util/env-test.cc
@@ -664,12 +664,27 @@ TEST_F(TestEnv, TestIsDirectory) {
   ASSERT_FALSE(is_dir);
 }
 
-// Regression test for KUDU-1776.
-TEST_F(TestEnv, TestIncreaseOpenFileLimit) {
-  int64_t limit_before = env_->GetOpenFileLimit();
-  env_->IncreaseOpenFileLimit();
-  int64_t limit_after = env_->GetOpenFileLimit();
-  ASSERT_GE(limit_after, limit_before) << "Failed to retain/increase open file 
limit";
+class ResourceLimitTypeTest : public TestEnv,
+  public 
::testing::WithParamInterface {};
+
+INSTANTIATE_TEST_CASE_P(ResourceLimitTypes,
+ResourceLimitTypeTest,
+
::testing::Values(Env::ResourceLimitType::OPEN_FILES_PER_PROCESS,
+  
Env::ResourceLimitType::RUNNING_THREADS_PER_EUID));
+
+// Regression test for KUDU-1798.
+TEST_P(ResourceLimitTypeTest, TestIncreaseLimit) {
+  // Increase the resource limit. It should either increase or remain the same.
+  Env::ResourceLimitType t = GetParam();
+  int64_t limit_before

[2/2] kudu git commit: KUDU-1913: cap number of threads on server-wide pools

2018-03-07 Thread adar 
KUDU-1913: cap number of threads on server-wide pools

The last piece of work is to establish an upper bound on the number of
threads that may be started in the Raft and Prepare server-wide threadpools.
Such caps will make it easier for admins to reason about appropriate values
for the configuration of the Kudu processes' RLIMIT_NPROC resource.

KUDU-1913 proposed a cap of "number of cores + number of disks", but a
lively Slack discussion yielded a better solution: set the cap at some
percentage of the process' RLIMIT_NPROC value. Given that the rest of Kudu
generally uses a constant number of threads, this should prevent spikes from
ever exceeding the RLIMIT_NPROC and crashing the server due to an election
storm. This patch implements a cap of 10% per pool and also provides a new
gflag as an "escape hatch" (in case we were horribly wrong).

Note: it's still possible for a massive number of "hot" replicas to exceed
RLIMIT_NPROC by virtue of each replica's log append thread, but the server
is more likely to run out of memory for MemRowSets before that happens.

Change-Id: I194907a7f8a483c9cba71eba8caed6bc6090f618
Reviewed-on: http://gerrit.cloudera.org:8080/9522
Tested-by: Kudu Jenkins
Reviewed-by: David Ribeiro Alves 
Reviewed-by: Todd Lipcon 


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/debcb8ea
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/debcb8ea
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/debcb8ea

Branch: refs/heads/master
Commit: debcb8ea21e95b57f5a9924b07acf43ca7a4a389
Parents: ede0cf0
Author: Adar Dembo 
Authored: Tue Mar 6 16:33:45 2018 -0800
Committer: Adar Dembo 
Committed: Thu Mar 8 02:43:34 2018 +

--
 src/kudu/kserver/kserver.cc | 68 +---
 1 file changed, 57 insertions(+), 11 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/debcb8ea/src/kudu/kserver/kserver.cc
--
diff --git a/src/kudu/kserver/kserver.cc b/src/kudu/kserver/kserver.cc
index e9ce303..b695979 100644
--- a/src/kudu/kserver/kserver.cc
+++ b/src/kudu/kserver/kserver.cc
@@ -17,17 +17,44 @@
 
 #include "kudu/kserver/kserver.h"
 
-#include 
+#include 
 #include 
+#include 
+#include 
 #include 
 #include 
 
+#include 
+#include 
+
+#include "kudu/fs/fs_manager.h"
+#include "kudu/gutil/strings/substitute.h"
 #include "kudu/rpc/messenger.h"
+#include "kudu/util/env.h"
+#include "kudu/util/flag_tags.h"
 #include "kudu/util/metrics.h"
 #include "kudu/util/status.h"
 #include "kudu/util/threadpool.h"
 
+DEFINE_int64(server_thread_pool_max_thread_count, -1,
+ "Maximum number of threads to allow in each server-wide thread "
+ "pool. If -1, Kudu will use 10% of its running thread per "
+ "effective uid resource limit as per getrlimit(). It is an error "
+ "to use a value of 0.");
+TAG_FLAG(server_thread_pool_max_thread_count, advanced);
+TAG_FLAG(server_thread_pool_max_thread_count, evolving);
+
+static bool ValidateThreadPoolThreadLimit(const char* /*flagname*/, int64_t 
value) {
+  if (value == 0) {
+LOG(ERROR) << "Invalid thread pool thread limit: cannot be 0";
+return false;
+  }
+  return true;
+}
+DEFINE_validator(server_thread_pool_max_thread_count, 
);
+
 using std::string;
+using strings::Substitute;
 
 namespace kudu {
 
@@ -56,6 +83,30 @@ METRIC_DEFINE_histogram(server, op_apply_run_time, 
"Operation Apply Run Time",
 "that operations consist of very large batches.",
 1000, 2);
 
+namespace {
+
+int64_t GetThreadPoolThreadLimit(Env* env) {
+  // Maximize this process' running thread limit first, if possible.
+  static std::once_flag once;
+  std::call_once(once, [&]() {
+
env->IncreaseResourceLimit(Env::ResourceLimitType::RUNNING_THREADS_PER_EUID);
+  });
+
+  int64_t rlimit = 
env->GetResourceLimit(Env::ResourceLimitType::RUNNING_THREADS_PER_EUID);
+  // See server_thread_pool_max_thread_count.
+  if (FLAGS_server_thread_pool_max_thread_count == -1) {
+return rlimit / 10;
+  }
+  LOG_IF(FATAL, FLAGS_server_thread_pool_max_thread_count > rlimit) <<
+  Substitute(
+  "Configured server-wide thread pool running thread limit "
+  "(server_thread_pool_max_thread_count) $0 exceeds euid running "
+  "thread limit (ulimit) $1",
+  FLAGS_server_thread_pool_max_thread_count, rlimit);
+  return FLAGS_server_thread_pool_max_thread_count;
+}
+
+} // anonymous namespace
 
 KuduServer::KuduServer(string name,
const ServerBaseOptions& options,
@@ -75,20 +126,15 @@ Status KuduServer::Init() {
 .set_metrics(std::move(metrics))

kudu git commit: Add CatalogManager::master_consensus() accessor

2018-03-07 Thread danburkert 
Repository: kudu
Updated Branches:
  refs/heads/master 74dc5da86 -> 954d9f71b


Add CatalogManager::master_consensus() accessor

A patch I'm working on is going to start calling
Master::GetMasterHostPorts from inside of CatalogManager::Init.
GetMasterHostPorts calls into the catalog manager, and checks that it is
initialized. To break this circular dependency this introduces a new
accessor for the master tablet RaftConsensus instance which becomes
available immediately after the tablet is initialized. A few call-sites
are switched over to this accessor instead of drilling into catalog
manager.

Change-Id: Ie6887900329e67222da129b4a1b532cfb0a364b4
Reviewed-on: http://gerrit.cloudera.org:8080/9541
Reviewed-by: Adar Dembo 
Reviewed-by: Alexey Serbin 
Tested-by: Kudu Jenkins


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/954d9f71
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/954d9f71
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/954d9f71

Branch: refs/heads/master
Commit: 954d9f71b50e2fae24a6e3acfe094b4d24dc5c2c
Parents: 74dc5da
Author: Dan Burkert 
Authored: Wed Mar 7 15:06:11 2018 -0800
Committer: Dan Burkert 
Committed: Thu Mar 8 00:39:23 2018 +

--
 src/kudu/integration-tests/security-master-auth-itest.cc |  9 ++---
 src/kudu/master/catalog_manager.cc   | 11 +++
 src/kudu/master/catalog_manager.h|  5 +
 src/kudu/master/master.cc|  9 ++---
 4 files changed, 20 insertions(+), 14 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/954d9f71/src/kudu/integration-tests/security-master-auth-itest.cc
--
diff --git a/src/kudu/integration-tests/security-master-auth-itest.cc 
b/src/kudu/integration-tests/security-master-auth-itest.cc
index c8eca38..9bdede1 100644
--- a/src/kudu/integration-tests/security-master-auth-itest.cc
+++ b/src/kudu/integration-tests/security-master-auth-itest.cc
@@ -22,16 +22,13 @@
 #include 
 
 #include "kudu/consensus/raft_consensus.h"
-#include "kudu/gutil/ref_counted.h"
 #include "kudu/master/catalog_manager.h"
 #include "kudu/master/master.h"
 #include "kudu/master/mini_master.h"
-#include "kudu/master/sys_catalog.h"
 #include "kudu/mini-cluster/internal_mini_cluster.h"
 #include "kudu/rpc/messenger.h"
 #include "kudu/security/tls_context.h"
 #include "kudu/security/token_verifier.h"
-#include "kudu/tablet/tablet_replica.h"
 #include "kudu/util/status.h"
 #include "kudu/util/test_macros.h"
 #include "kudu/util/test_util.h"
@@ -84,8 +81,7 @@ TEST_F(SecurityMasterAuthTest, FollowerCertificates) {
 ASSERT_TRUE(tls.has_cert());
   }
 
-  auto* consensus = cluster_->mini_master(0)->master()->catalog_manager()->
-  sys_catalog()->tablet_replica()->consensus();
+  auto consensus = 
cluster_->mini_master(0)->master()->catalog_manager()->master_consensus();
   ASSERT_OK(consensus->StartElection(
   RaftConsensus::ELECT_EVEN_IF_LEADER_IS_ALIVE,
   RaftConsensus::EXTERNAL_REQUEST));
@@ -105,8 +101,7 @@ TEST_F(SecurityMasterAuthTest, FollowerCertificates) {
 // the rest have always been followers. This is a test to cover regressions of
 // KUDU-2319, if any.
 TEST_F(SecurityMasterAuthTest, FollowerTokenVerificationKeys) {
-  auto* consensus = cluster_->mini_master(0)->master()->catalog_manager()->
-  sys_catalog()->tablet_replica()->consensus();
+  auto consensus = 
cluster_->mini_master(0)->master()->catalog_manager()->master_consensus();
   ASSERT_OK(consensus->StartElection(
   RaftConsensus::ELECT_EVEN_IF_LEADER_IS_ALIVE,
   RaftConsensus::EXTERNAL_REQUEST));

http://git-wip-us.apache.org/repos/asf/kudu/blob/954d9f71/src/kudu/master/catalog_manager.cc
--
diff --git a/src/kudu/master/catalog_manager.cc 
b/src/kudu/master/catalog_manager.cc
index 974927e..2aa09af 100644
--- a/src/kudu/master/catalog_manager.cc
+++ b/src/kudu/master/catalog_manager.cc
@@ -3755,6 +3755,17 @@ Status CatalogManager::ProcessTabletReport(
   return Status::OK();
 }
 
+std::shared_ptr CatalogManager::master_consensus() const {
+  // CatalogManager::InitSysCatalogAsync takes lock_ in exclusive mode in order
+  // to initialize sys_catalog_, so it's sufficient to take lock_ in shared 
mode
+  // here to protect access to sys_catalog_.
+  shared_lock l(lock_);
+  if (!sys_catalog_) {
+return nullptr;
+  }
+  return sys_catalog_->tablet_replica()->shared_consensus();
+}
+
 void CatalogManager::SendAlterTableRequest(const scoped_refptr& 
table) {
   vector tablets;
   table->GetAllTablets();

kudu git commit: Fix RUN_FLAKY_ONLY build

2018-03-07 Thread todd 
Repository: kudu
Updated Branches:
  refs/heads/master e6aedc99d -> 74dc5da86


Fix RUN_FLAKY_ONLY build

Now that our tests have shard numbers appended (foo-test.)
according to ctest, the RUN_FLAKY_ONLY build regex was no longer
applying properly. It was converting the flaky test 'foo-test' to the
regex '^foo-test$' which would not match any of the shards.

This simply removes the '$' from the end of the regex. I attempted to
instead add a '(.\d+)?' to the regex, but ctest has a hard limit on the
number of '()' pairs in a regex so it was failing to compile the regex.
It's also not possible to pass multiple separate regexes on the command
line. Given that we never have test names where one is a prefix of
another, I think this is probably fine.

Change-Id: I3574bb88c2f3c465e3c659b80b122c1109053648
Reviewed-on: http://gerrit.cloudera.org:8080/9533
Reviewed-by: Adar Dembo 
Tested-by: Kudu Jenkins


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/74dc5da8
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/74dc5da8
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/74dc5da8

Branch: refs/heads/master
Commit: 74dc5da86ab2e5c8bfe524599d4a9e7a04d3cd27
Parents: e6aedc9
Author: Todd Lipcon 
Authored: Wed Mar 7 11:51:19 2018 -0800
Committer: Todd Lipcon 
Committed: Wed Mar 7 20:17:15 2018 +

--
 build-support/jenkins/build-and-test.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/74dc5da8/build-support/jenkins/build-and-test.sh
--
diff --git a/build-support/jenkins/build-and-test.sh 
b/build-support/jenkins/build-and-test.sh
index 66cc2ae..12a2460 100755
--- a/build-support/jenkins/build-and-test.sh
+++ b/build-support/jenkins/build-and-test.sh
@@ -307,7 +307,7 @@ if [ "$RUN_FLAKY_ONLY" == "1" ] ; then
   fi
   test_regex=$(perl -e '
 chomp(my @lines = <>);
-print join("|", map { "^" . quotemeta($_) . "\$" } @lines);
+print join("|", map { "^" . quotemeta($_) } @lines);
' $BUILD_ROOT/flaky-tests.txt)
   if [ -z "$test_regex" ]; then
 echo No tests are flaky.

[2/2] kudu git commit: KUDU-721: [Python] Add DECIMAL column type support

2018-03-07 Thread dralves 
KUDU-721: [Python] Add DECIMAL column type support

This patch adds basic support to the Python client to
create, read, and write tables with DECIMAL columns.

Change-Id: I8e0855100ab1ea891f990931ec94d0b98c0dece1
Reviewed-on: http://gerrit.cloudera.org:8080/9496
Tested-by: Kudu Jenkins
Reviewed-by: David Ribeiro Alves 


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/e6aedc99
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/e6aedc99
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/e6aedc99

Branch: refs/heads/master
Commit: e6aedc99d25ba8bc8f1fc37259404d55c802953f
Parents: 2b68040
Author: Grant Henke 
Authored: Mon Mar 5 14:23:08 2018 -0600
Committer: David Ribeiro Alves 
Committed: Wed Mar 7 19:46:57 2018 +

--
 python/kudu/__init__.py |   2 +-
 python/kudu/client.pyx  |  20 +-
 python/kudu/libkudu_client.pxd  |  36 ++-
 python/kudu/schema.pxd  |   8 +++
 python/kudu/schema.pyx  | 106 +--
 python/kudu/tests/test_scanner.py   |   4 ++
 python/kudu/tests/test_scantoken.py |   4 ++
 python/kudu/tests/test_schema.py|  40 
 python/kudu/tests/test_util.py  |  80 +++
 python/kudu/tests/util.py   |  14 +++-
 python/kudu/util.py |  65 +++
 11 files changed, 368 insertions(+), 11 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/e6aedc99/python/kudu/__init__.py
--
diff --git a/python/kudu/__init__.py b/python/kudu/__init__.py
index 8f0c8d9..75260d6 100644
--- a/python/kudu/__init__.py
+++ b/python/kudu/__init__.py
@@ -36,7 +36,7 @@ from kudu.errors import (KuduException, KuduBadStatus, 
KuduNotFound,  # noqa
 
 from kudu.schema import (int8, int16, int32, int64, string_ as string,  # noqa
  double_ as double, float_, float_ as float, binary,
- unixtime_micros, bool_ as bool,
+ unixtime_micros, bool_ as bool, decimal,
  KuduType,
  SchemaBuilder, ColumnSpec, Schema, ColumnSchema,
  COMPRESSION_DEFAULT,

http://git-wip-us.apache.org/repos/asf/kudu/blob/e6aedc99/python/kudu/client.pyx
--
diff --git a/python/kudu/client.pyx b/python/kudu/client.pyx
index 0b28856..34cb53a 100644
--- a/python/kudu/client.pyx
+++ b/python/kudu/client.pyx
@@ -29,7 +29,8 @@ from libkudu_client cimport *
 from kudu.compat import tobytes, frombytes, dict_iter
 from kudu.schema cimport Schema, ColumnSchema, ColumnSpec, KuduValue, KuduType
 from kudu.errors cimport check_status
-from kudu.util import to_unixtime_micros, from_unixtime_micros, from_hybridtime
+from kudu.util import to_unixtime_micros, from_unixtime_micros, \
+from_hybridtime, to_unscaled_decimal, from_unscaled_decimal
 from errors import KuduException
 
 import six
@@ -64,7 +65,8 @@ cdef dict _type_names = {
 KUDU_FLOAT : "KUDU_FLOAT",
 KUDU_DOUBLE : "KUDU_DOUBLE",
 KUDU_BINARY : "KUDU_BINARY",
-KUDU_UNIXTIME_MICROS : "KUDU_UNIXTIME_MICROS"
+KUDU_UNIXTIME_MICROS : "KUDU_UNIXTIME_MICROS",
+KUDU_DECIMAL : "KUDU_DECIMAL"
 }
 
 # Range Partition Bound Type enums
@@ -1314,6 +1316,15 @@ cdef class Row:
 check_status(self.row.GetUnixTimeMicros(i, ))
 return val
 
+cdef inline __get_unscaled_decimal(self, int i):
+cdef int128_t val
+check_status(self.row.GetUnscaledDecimal(i, ))
+return val
+
+cdef inline get_decimal(self, int i):
+scale = 
self.parent.batch.projection_schema().Column(i).type_attributes().scale()
+return from_unscaled_decimal(self.__get_unscaled_decimal(i), scale)
+
 cdef inline get_slot(self, int i):
 cdef:
 Status s
@@ -1339,6 +1350,8 @@ cdef class Row:
 return self.get_binary(i)
 elif t == KUDU_UNIXTIME_MICROS:
 return from_unixtime_micros(self.get_unixtime_micros(i))
+elif t == KUDU_DECIMAL:
+return self.get_decimal(i)
 else:
 raise TypeError("Cannot get kudu type <{0}>"
 .format(_type_names[t]))
@@ -2451,6 +2464,9 @@ cdef class PartialRow:
 elif t == KUDU_UNIXTIME_MICROS:
 check_status(self.row.SetUnixTimeMicros(i, 
 to_unixtime_micros(value)))
+elif t == KUDU_DECIMAL:
+check_status(self.row.SetUnscaledDecimal(i, 
+to_unscaled_decimal(value)))
 else:
 raise TypeError("Cannot set kudu type