kudu git commit: hms-tool: filter non-Kudu tables in the HMS
Repository: kudu
Updated Branches:
refs/heads/master f62bbdd64 -> d074de85d
hms-tool: filter non-Kudu tables in the HMS
Renames HmsCatalog::RetrieveTables to GetKuduTables and changes the
semantics such that only Kudu tables are returned. The only caller was
the HMS tool, and it only needs to inspect Kudu tables.
Special APIs are added to the HMS client and HMS catalog layers that
allow sending a filter to the HMS to strip out non-Kudu tables when
listing tables, as well as a bulk-get API. The combination of these APIs
should be significantly more efficient than issuing a get for every
single table in the HMS and doing Kudu-side filtering.
Also included are some style and formatting fixups in HMS tool.
This patch includes no functional changes to the HMS tool.
Change-Id: I5f83d2e705ea6910a9aa0a1eda0d30b5feb2607b
Reviewed-on: http://gerrit.cloudera.org:8080/10934
Reviewed-by: Adar Dembo
Tested-by: Kudu Jenkins
Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/d074de85
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/d074de85
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/d074de85
Branch: refs/heads/master
Commit: d074de85dd19697444bb48eabe5b5c577a579d07
Parents: f62bbdd
Author: Dan Burkert
Authored: Tue Jul 3 17:12:59 2018 -0700
Committer: Dan Burkert
Committed: Fri Jul 13 02:34:34 2018 +
--
src/kudu/hms/hms_catalog-test.cc| 37 +-
src/kudu/hms/hms_catalog.cc | 27 +--
src/kudu/hms/hms_catalog.h | 12 ++-
src/kudu/hms/hms_client-test.cc | 49 ++---
src/kudu/hms/hms_client.cc | 30 +++-
src/kudu/hms/hms_client.h | 19 -
src/kudu/integration-tests/master_hms-itest.cc | 2 +-
.../mini-cluster/external_mini_cluster-test.cc | 2 +-
src/kudu/tools/kudu-tool-test.cc| 6 +-
src/kudu/tools/tool_action_hms.cc | 77 +---
10 files changed, 167 insertions(+), 94 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kudu/blob/d074de85/src/kudu/hms/hms_catalog-test.cc
--
diff --git a/src/kudu/hms/hms_catalog-test.cc b/src/kudu/hms/hms_catalog-test.cc
index 472f3a8..4574402 100644
--- a/src/kudu/hms/hms_catalog-test.cc
+++ b/src/kudu/hms/hms_catalog-test.cc
@@ -396,36 +396,39 @@ TEST_F(HmsCatalogTest, TestExternalTable) {
NO_FATALS(CheckTableDoesNotExist("default", "bogus_table_name"));
}
-TEST_F(HmsCatalogTest, TestRetrieveTables) {
+TEST_F(HmsCatalogTest, TestGetKuduTables) {
const string kHmsDatabase = "db";
const string kManagedTableName = "managed_table";
const string kExternalTableName = "external_table";
+ const string kTableName = "external_table";
const string kNonKuduTableName = "non_kudu_table";
- // Create a Impala managed table, a external table and a non Kudu table.
+ // Create a legacy Impala managed table, a legacy Impala external table, a
+ // Kudu table, and a non Kudu table.
hive::Database db;
- db.name = kHmsDatabase;
+ db.name = "db";
ASSERT_OK(hms_client_->CreateDatabase(db));
- ASSERT_OK(CreateLegacyTable(kHmsDatabase,
- kManagedTableName,
- HmsClient::kManagedTable));
+ ASSERT_OK(CreateLegacyTable("db", "managed_table",
HmsClient::kManagedTable));
hive::Table table;
- ASSERT_OK(hms_client_->GetTable(kHmsDatabase, kManagedTableName, ));
- ASSERT_OK(CreateLegacyTable(kHmsDatabase,
- kExternalTableName,
- HmsClient::kExternalTable));
- ASSERT_OK(hms_client_->GetTable(kHmsDatabase, kExternalTableName, ));
+ ASSERT_OK(hms_client_->GetTable(kHmsDatabase, "managed_table", ));
+ ASSERT_OK(CreateLegacyTable("db", "external_table",
HmsClient::kExternalTable));
+ ASSERT_OK(hms_client_->GetTable("db", "external_table", ));
+
+ ASSERT_OK(hms_catalog_->CreateTable("fake-id", "db.table", Schema()));
hive::Table non_kudu_table;
- non_kudu_table.dbName = kHmsDatabase;
- non_kudu_table.tableName = kNonKuduTableName;
+ non_kudu_table.dbName = "db";
+ non_kudu_table.tableName = "non_kudu_table";
ASSERT_OK(hms_client_->CreateTable(non_kudu_table));
- ASSERT_OK(hms_client_->GetTable(kHmsDatabase, kNonKuduTableName, ));
+ ASSERT_OK(hms_client_->GetTable("db", "non_kudu_table", ));
// Retrieve all tables and ensure all entries are found.
- vector hms_tables;
- ASSERT_OK(hms_catalog_->RetrieveTables(_tables));
- ASSERT_EQ(3, hms_tables.size());
+ vector kudu_tables;
+ ASSERT_OK(hms_catalog_->GetKuduTables(_tables));
+ ASSERT_EQ(3, kudu_tables.size());
+ for (const auto& kudu_table : kudu_tables) {
+ASSERT_FALSE(kudu_table.tableName ==
[1/2] kudu git commit: [Java] Fix the use of resource files in tests
Repository: kudu
Updated Branches:
refs/heads/master 978cf3912 -> f62bbdd64
[Java] Fix the use of resource files in tests
This is preliminary work needed to run the java tests
on dist-test.
Our tests currently reference files/resources that are
not available in the output test artifacts. This means
our tests work when they are run from source in the
expected location but fail when they are run from the
compiled artifacts.
This fixes those failing tests by discovering the correct
location via Class.getResource and ensuring all needed
files are packaged in the resources directory.
Change-Id: Ia3a90bac67652b58f2eef802c509983c68d8a299
Reviewed-on: http://gerrit.cloudera.org:8080/10924
Reviewed-by: Andrew Wong
Tested-by: Grant Henke
Reviewed-by: Mike Percy
Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/fff3cf09
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/fff3cf09
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/fff3cf09
Branch: refs/heads/master
Commit: fff3cf0923782cab8a43e081caeccc88a53f11f9
Parents: 978cf39
Author: Grant Henke
Authored: Wed Jul 11 14:17:04 2018 -0500
Committer: Grant Henke
Committed: Thu Jul 12 17:14:53 2018 +
--
java/kudu-flume-sink/build.gradle | 6 ++
java/kudu-flume-sink/pom.xml| 3 +++
.../avro/testAvroKuduOperationsProducer.avsc| 14 -
.../sink/AvroKuduOperationsProducerTest.java| 22 ++--
.../testAvroKuduOperationsProducer.avsc | 14 +
.../spark/tools/TestImportExportFiles.scala | 10 +
6 files changed, 40 insertions(+), 29 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kudu/blob/fff3cf09/java/kudu-flume-sink/build.gradle
--
diff --git a/java/kudu-flume-sink/build.gradle
b/java/kudu-flume-sink/build.gradle
index 5767103..e09f0ae 100644
--- a/java/kudu-flume-sink/build.gradle
+++ b/java/kudu-flume-sink/build.gradle
@@ -33,4 +33,10 @@ dependencies {
testCompile project(path: ":kudu-client", configuration: "shadowTest")
testCompile libs.junit
+}
+
+// Configure the Avro plugin to compile the schemas in the
+// resource directory so the schemas are available in tests.
+generateTestAvroJava {
+ source("src/test/resources")
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kudu/blob/fff3cf09/java/kudu-flume-sink/pom.xml
--
diff --git a/java/kudu-flume-sink/pom.xml b/java/kudu-flume-sink/pom.xml
index aa8a859..7b664fb 100644
--- a/java/kudu-flume-sink/pom.xml
+++ b/java/kudu-flume-sink/pom.xml
@@ -126,6 +126,9 @@
schema
+
+
${basedir}/src/test/resources
true
http://git-wip-us.apache.org/repos/asf/kudu/blob/fff3cf09/java/kudu-flume-sink/src/test/avro/testAvroKuduOperationsProducer.avsc
--
diff --git
a/java/kudu-flume-sink/src/test/avro/testAvroKuduOperationsProducer.avsc
b/java/kudu-flume-sink/src/test/avro/testAvroKuduOperationsProducer.avsc
deleted file mode 100644
index 6bcf6d2..000
--- a/java/kudu-flume-sink/src/test/avro/testAvroKuduOperationsProducer.avsc
+++ /dev/null
@@ -1,14 +0,0 @@
-{"namespace": "org.apache.kudu.flume.sink",
- "type": "record",
- "name": "AvroKuduOperationsProducerTestRecord",
- "fields": [
-{"name": "key", "type": "int"},
-{"name": "longField", "type": "long"},
-{"name": "doubleField", "type": "double"},
-{"name": "nullableField", "type": ["string", "null"]},
-{"name": "stringField", "type": "string"},
-{"name": "decimalField", "type": {
-"type": "bytes", "logicalType": "decimal", "precision": 9, "scale": 1}
-}
- ]
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kudu/blob/fff3cf09/java/kudu-flume-sink/src/test/java/org/apache/kudu/flume/sink/AvroKuduOperationsProducerTest.java
--
diff --git
a/java/kudu-flume-sink/src/test/java/org/apache/kudu/flume/sink/AvroKuduOperationsProducerTest.java
b/java/kudu-flume-sink/src/test/java/org/apache/kudu/flume/sink/AvroKuduOperationsProducerTest.java
index 8e3c8dc..42c94cd 100644
---
a/java/kudu-flume-sink/src/test/java/org/apache/kudu/flume/sink/AvroKuduOperationsProducerTest.java
+++
b/java/kudu-flume-sink/src/test/java/org/apache/kudu/flume/sink/AvroKuduOperationsProducerTest.java
@@ -33,8 +33,9 @@ import static org.junit.Assert.assertEquals;
import java.io.ByteArrayOutputStream;
import java.io.File;
-import java.io.IOException;
import java.math.BigDecimal;
+import
[2/2] kudu git commit: Test for backup/restore multiple tables
Test for backup/restore multiple tables
This is a prerequisite for testing parallel backup/restore and error
handling.
- Extracted `createTable` method so it can be re-used
- Added `tableName` argument to `insertRows` so it can be re-used
Change-Id: I439e0d1c6904395ad382e3f059846b3b03a79af4
Reviewed-on: http://gerrit.cloudera.org:8080/10899
Tested-by: Kudu Jenkins
Reviewed-by: Adar Dembo
Reviewed-by: Grant Henke
Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/f62bbdd6
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/f62bbdd6
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/f62bbdd6
Branch: refs/heads/master
Commit: f62bbdd64d1c039b91edbd1d55a1607abd023f43
Parents: fff3cf0
Author: Tony Foerster
Authored: Tue Jul 10 07:08:52 2018 -0500
Committer: Grant Henke
Committed: Thu Jul 12 19:17:09 2018 +
--
.../org/apache/kudu/backup/TestKuduBackup.scala | 28 +---
.../kudu/spark/kudu/DefaultSourceTest.scala | 2 +-
.../kudu/spark/kudu/KuduContextTest.scala | 4 +--
.../apache/kudu/spark/kudu/KuduRDDTest.scala| 2 +-
.../apache/kudu/spark/kudu/TestContext.scala| 24 ++---
5 files changed, 42 insertions(+), 18 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kudu/blob/f62bbdd6/java/kudu-backup/src/test/scala/org/apache/kudu/backup/TestKuduBackup.scala
--
diff --git
a/java/kudu-backup/src/test/scala/org/apache/kudu/backup/TestKuduBackup.scala
b/java/kudu-backup/src/test/scala/org/apache/kudu/backup/TestKuduBackup.scala
index 2e2e589..1c166bd 100644
---
a/java/kudu-backup/src/test/scala/org/apache/kudu/backup/TestKuduBackup.scala
+++
b/java/kudu-backup/src/test/scala/org/apache/kudu/backup/TestKuduBackup.scala
@@ -42,7 +42,7 @@ class TestKuduBackup extends FunSuite with TestContext with
Matchers {
val log: Logger = LoggerFactory.getLogger(getClass)
test("Simple Backup and Restore") {
-insertRows(100) // Insert data into the default test table.
+insertRows(table, 100) // Insert data into the default test table.
backupAndRestore(tableName)
@@ -93,6 +93,26 @@ class TestKuduBackup extends FunSuite with TestContext with
Matchers {
assertTrue(partitionSchemasMatch(tA.getPartitionSchema,
tB.getPartitionSchema))
}
+ test("Backup and Restore Multiple Tables") {
+val numRows = 1
+val table1Name = "table1"
+val table2Name = "table2"
+
+val table1 = kuduClient.createTable(table1Name, schema, tableOptions)
+val table2 = kuduClient.createTable(table2Name, schema, tableOptions)
+
+insertRows(table1, numRows)
+insertRows(table2, numRows)
+
+backupAndRestore(table1Name, table2Name)
+
+val rdd1 = kuduContext.kuduRDD(ss.sparkContext, s"$table1Name-restore",
List("key"))
+assertResult(numRows)(rdd1.count())
+
+val rdd2 = kuduContext.kuduRDD(ss.sparkContext, s"$table2Name-restore",
List("key"))
+assertResult(numRows)(rdd2.count())
+ }
+
// TODO: Move to a Schema equals/equivalent method.
def schemasMatch(before: Schema, after: Schema): Boolean = {
if (before eq after) return true
@@ -299,14 +319,14 @@ class TestKuduBackup extends FunSuite with TestContext
with Matchers {
}
}
- def backupAndRestore(tableName: String): Unit = {
+ def backupAndRestore(tableNames: String*): Unit = {
val dir = Files.createTempDirectory("backup")
val path = dir.toUri.toString
-val backupOptions = new KuduBackupOptions(Seq(tableName), path,
miniCluster.getMasterAddresses)
+val backupOptions = new KuduBackupOptions(tableNames, path,
miniCluster.getMasterAddresses)
KuduBackup.run(backupOptions, ss)
-val restoreOptions = new KuduRestoreOptions(Seq(tableName), path,
miniCluster.getMasterAddresses)
+val restoreOptions = new KuduRestoreOptions(tableNames, path,
miniCluster.getMasterAddresses)
KuduRestore.run(restoreOptions, ss)
FileUtils.deleteDirectory(dir.toFile)
http://git-wip-us.apache.org/repos/asf/kudu/blob/f62bbdd6/java/kudu-spark/src/test/scala/org/apache/kudu/spark/kudu/DefaultSourceTest.scala
--
diff --git
a/java/kudu-spark/src/test/scala/org/apache/kudu/spark/kudu/DefaultSourceTest.scala
b/java/kudu-spark/src/test/scala/org/apache/kudu/spark/kudu/DefaultSourceTest.scala
index 4e7d5a4..ec4d37c 100644
---
a/java/kudu-spark/src/test/scala/org/apache/kudu/spark/kudu/DefaultSourceTest.scala
+++
b/java/kudu-spark/src/test/scala/org/apache/kudu/spark/kudu/DefaultSourceTest.scala
@@ -41,7 +41,7 @@ class DefaultSourceTest extends FunSuite with TestContext
with BeforeAndAfterEac
override def beforeEach(): Unit = {
super.beforeEach()
-rows = insertRows(rowCount)
+
kudu git commit: rpcz: print timeout units (ms) when logging
Repository: kudu
Updated Branches:
refs/heads/master 2692ac4ad -> 978cf3912
rpcz: print timeout units (ms) when logging
Also print in human-readable format, which is easier to read for large
values, for both the duration and the timeout. Since the human-readable
version leaves a space between the value and the unit, do the same.
Change-Id: I043d25b83cb4d2470cf0d9d3945e8ace633d9a5a
Reviewed-on: http://gerrit.cloudera.org:8080/10915
Tested-by: Kudu Jenkins
Reviewed-by: Dan Burkert
Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/978cf391
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/978cf391
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/978cf391
Branch: refs/heads/master
Commit: 978cf39124e1d5b637618384afd985cd3a002526
Parents: 2692ac4
Author: Mike Percy
Authored: Fri Jul 6 13:10:07 2018 -0700
Committer: Mike Percy
Committed: Thu Jul 12 16:37:53 2018 +
--
src/kudu/rpc/rpcz_store.cc | 8 ++--
1 file changed, 6 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kudu/blob/978cf391/src/kudu/rpc/rpcz_store.cc
--
diff --git a/src/kudu/rpc/rpcz_store.cc b/src/kudu/rpc/rpcz_store.cc
index 2f0e9c8..f7c7b3b 100644
--- a/src/kudu/rpc/rpcz_store.cc
+++ b/src/kudu/rpc/rpcz_store.cc
@@ -32,6 +32,7 @@
#include "kudu/gutil/port.h"
#include "kudu/gutil/ref_counted.h"
+#include "kudu/gutil/strings/human_readable.h"
#include "kudu/gutil/strings/stringpiece.h"
#include "kudu/gutil/walltime.h"
#include "kudu/rpc/inbound_call.h"
@@ -248,8 +249,11 @@ void RpczStore::LogTrace(InboundCall* call) {
if (duration_ms > log_threshold) {
// TODO: consider pushing this onto another thread since it may be slow.
// The traces may also be too large to fit in a log message.
- LOG(WARNING) << call->ToString() << " took " << duration_ms << "ms
(client timeout "
- << call->header_.timeout_millis() << ").";
+ int64_t timeout_ms = call->header_.timeout_millis();
+ LOG(WARNING) << call->ToString() << " took " << duration_ms << " ms "
+ << "(" <<
HumanReadableElapsedTime::ToShortString(duration_ms * .001) << "). "
+ << "Client timeout " << timeout_ms << " ms "
+ << "(" <<
HumanReadableElapsedTime::ToShortString(timeout_ms * .001) << ")";
string s = call->trace()->DumpToString();
if (!s.empty()) {
LOG(WARNING) << "Trace:\n" << s;
