date:20180723

[2/2] kudu git commit: [log-test] clean-up on WAL entry test operations

2018-07-23 Thread alexey

[log-test] clean-up on WAL entry test operations

Change-Id: I01ef7798c63c18cb03df5969fc73bbb0ec65111b
Reviewed-on: http://gerrit.cloudera.org:8080/11028
Reviewed-by: Adar Dembo 
Tested-by: Alexey Serbin 


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/5b09a693
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/5b09a693
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/5b09a693

Branch: refs/heads/master
Commit: 5b09a693dd651203512abf9faaddcc21aad92e59
Parents: 6b429e8
Author: Alexey Serbin 
Authored: Mon Jul 23 18:20:47 2018 -0700
Committer: Alexey Serbin 
Committed: Tue Jul 24 04:42:49 2018 +

--
 src/kudu/consensus/log-test-base.h   | 97 ++-
 src/kudu/consensus/log-test.cc   | 22 +++---
 src/kudu/tablet/tablet_bootstrap-test.cc | 69 ++-
 3 files changed, 95 insertions(+), 93 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/5b09a693/src/kudu/consensus/log-test-base.h
--
diff --git a/src/kudu/consensus/log-test-base.h 
b/src/kudu/consensus/log-test-base.h
index 5239ea2..987b769 100644
--- a/src/kudu/consensus/log-test-base.h
+++ b/src/kudu/consensus/log-test-base.h
@@ -65,10 +65,10 @@ constexpr bool APPEND_ASYNC = false;
 // If 'size' is not NULL, increments it by the expected increase in log size.
 // Increments 'op_id''s index once for each operation logged.
 inline Status AppendNoOpsToLogSync(const scoped_refptr& clock,
-Log* log,
-consensus::OpId* op_id,
-int count,
-int* size = NULL) {
+   Log* log,
+   consensus::OpId* op_id,
+   int count,
+   int* size = nullptr) {
 
   std::vector replicates;
   for (int i = 0; i < count; i++) {
@@ -103,9 +103,9 @@ inline Status AppendNoOpsToLogSync(const 
scoped_refptr& clock,
 }
 
 inline Status AppendNoOpToLogSync(const scoped_refptr& clock,
-   Log* log,
-   consensus::OpId* op_id,
-   int* size = NULL) {
+  Log* log,
+  consensus::OpId* op_id,
+  int* size = nullptr) {
   return AppendNoOpsToLogSync(clock, log, op_id, 1, size);
 }
 
@@ -145,11 +145,11 @@ class LogTestBase : public KuduTest {
   typedef std::pair DeltaId;
 
   LogTestBase()
-: schema_(GetSimpleTestSchema()),
-  log_anchor_registry_(new LogAnchorRegistry()) {
+  : schema_(GetSimpleTestSchema()),
+log_anchor_registry_(new LogAnchorRegistry) {
   }
 
-  virtual void SetUp() OVERRIDE {
+  void SetUp() override {
 KuduTest::SetUp();
 current_index_ = kStartIndex;
 fs_manager_.reset(new FsManager(env_, GetTestPath("fs_root")));
@@ -162,7 +162,7 @@ class LogTestBase : public KuduTest {
 ASSERT_OK(clock_->Init());
   }
 
-  virtual void TearDown() OVERRIDE {
+  void TearDown() override {
 KuduTest::TearDown();
 STLDeleteElements(_);
   }
@@ -209,14 +209,15 @@ class LogTestBase : public KuduTest {
   }
 
   // Appends a batch with size 2 (1 insert, 1 mutate) to the log.
-  void AppendReplicateBatch(const consensus::OpId& opid, bool sync = 
APPEND_SYNC) {
+  Status AppendReplicateBatch(const consensus::OpId& opid,
+  bool sync = APPEND_SYNC) {
 consensus::ReplicateRefPtr replicate =
 make_scoped_refptr_replicate(new consensus::ReplicateMsg());
 replicate->get()->set_op_type(consensus::WRITE_OP);
 replicate->get()->mutable_id()->CopyFrom(opid);
 replicate->get()->set_timestamp(clock_->Now().ToUint64());
 tserver::WriteRequestPB* batch_request = 
replicate->get()->mutable_write_request();
-ASSERT_OK(SchemaToPB(schema_, batch_request->mutable_schema()));
+RETURN_NOT_OK(SchemaToPB(schema_, batch_request->mutable_schema()));
 AddTestRowToPB(RowOperationsPB::INSERT, schema_,
opid.index(),
0,
@@ -228,22 +229,21 @@ class LogTestBase : public KuduTest {
"this is a test mutate",
batch_request->mutable_row_operations());
 batch_request->set_tablet_id(kTestTablet);
-AppendReplicateBatch(replicate, sync);
+return AppendReplicateBatch(replicate, sync);
   }
 
   // Appends the provided batch to the log.
-  void AppendReplicateBatch(const consensus::ReplicateRefPtr& replicate,
-bool sync = APPEND_SYNC) {
+  Status AppendReplicateBatch(const consensus::ReplicateRefPtr& replicate,
+  bool sync = APPEND_SYNC) {

[1/2] kudu git commit: KUDU-2509 fix use-after-free in case of WAL replay error

2018-07-23 Thread alexey

Repository: kudu
Updated Branches:
  refs/heads/master ec1e47f41 -> 5b09a693d


KUDU-2509 fix use-after-free in case of WAL replay error

Fixed use-after-free mistake in case of a failure to apply a pending
commit message from the WAL while bootstrapping a tablet.

Also, a repro scenario to expose the use-after-free condition is added.
Prior to the fix, the repro scenario would crash with SIGSEGV on Linux
or with SIGBUS on OS X (at least for DEBUG builds).

Change-Id: I11373b1cc34d9e2e0181bee2d3841b49022218ed
Reviewed-on: http://gerrit.cloudera.org:8080/10997
Tested-by: Alexey Serbin 
Reviewed-by: Adar Dembo 


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/6b429e8a
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/6b429e8a
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/6b429e8a

Branch: refs/heads/master
Commit: 6b429e8a42ad9fb12a97cc26e33ca19ac2626533
Parents: ec1e47f
Author: Alexey Serbin 
Authored: Thu Jul 19 21:05:54 2018 -0700
Committer: Alexey Serbin 
Committed: Tue Jul 24 04:40:20 2018 +

--
 src/kudu/tablet/tablet_bootstrap-test.cc | 75 ++-
 src/kudu/tablet/tablet_bootstrap.cc  |  2 +-
 2 files changed, 74 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/6b429e8a/src/kudu/tablet/tablet_bootstrap-test.cc
--
diff --git a/src/kudu/tablet/tablet_bootstrap-test.cc 
b/src/kudu/tablet/tablet_bootstrap-test.cc
index 0b80c20..f5b2668 100644
--- a/src/kudu/tablet/tablet_bootstrap-test.cc
+++ b/src/kudu/tablet/tablet_bootstrap-test.cc
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#include "kudu/consensus/log-test-base.h"
+#include "kudu/tablet/tablet_bootstrap.h"
 
 #include 
 #include 
@@ -35,7 +35,9 @@
 #include "kudu/clock/logical_clock.h"
 #include "kudu/common/common.pb.h"
 #include "kudu/common/iterator.h"
+#include "kudu/common/partial_row.h"
 #include "kudu/common/partition.h"
+#include "kudu/common/row_operations.h"
 #include "kudu/common/schema.h"
 #include "kudu/common/timestamp.h"
 #include "kudu/common/wire_protocol-test-util.h"
@@ -45,6 +47,7 @@
 #include "kudu/consensus/consensus.pb.h"
 #include "kudu/consensus/consensus_meta.h"
 #include "kudu/consensus/consensus_meta_manager.h"
+#include "kudu/consensus/log-test-base.h"
 #include "kudu/consensus/log.h"
 #include "kudu/consensus/log_anchor_registry.h"
 #include "kudu/consensus/log_reader.h"
@@ -67,7 +70,6 @@
 #include "kudu/tablet/tablet-test-util.h"
 #include "kudu/tablet/tablet.h"
 #include "kudu/tablet/tablet.pb.h"
-#include "kudu/tablet/tablet_bootstrap.h"
 #include "kudu/tablet/tablet_metadata.h"
 #include "kudu/tablet/tablet_replica.h"
 #include "kudu/tserver/tserver.pb.h"
@@ -675,5 +677,74 @@ TEST_F(BootstrapTest, 
TestConsensusOnlyOperationOutOfOrderTimestamp) {
   ASSERT_EQ(1, results.size());
 }
 
+// Regression test for KUDU-2509. There was a use-after-free bug that sometimes
+// lead to SIGSEGV while replaying the WAL. This scenario would crash or
+// at least UB sanitizer would report a warning if such condition exists.
+TEST_F(BootstrapTest, TestKudu2509) {
+  ASSERT_OK(BuildLog());
+
+  consensus::ReplicateRefPtr replicate = 
consensus::make_scoped_refptr_replicate(
+  new consensus::ReplicateMsg());
+  replicate->get()->set_op_type(consensus::WRITE_OP);
+  tserver::WriteRequestPB* batch_request = 
replicate->get()->mutable_write_request();
+  ASSERT_OK(SchemaToPB(schema_, batch_request->mutable_schema()));
+  batch_request->set_tablet_id(log::kTestTablet);
+
+  // This appends Insert(1) with op 10.10
+  const OpId insert_opid = MakeOpId(10, 10);
+  replicate->get()->mutable_id()->CopyFrom(insert_opid);
+  replicate->get()->set_timestamp(clock_->Now().ToUint64());
+  AddTestRowToPB(RowOperationsPB::INSERT, schema_, 10, 1,
+ "this is a test insert", 
batch_request->mutable_row_operations());
+  NO_FATALS(AppendReplicateBatch(replicate));
+
+  // This appends Mutate(1) with op 10.11. The operation would try to update
+  // a row having an extra column. This should fail since there hasn't been
+  // corresponding DDL operation committed yet.
+  const OpId mutate_opid = MakeOpId(10, 11);
+  batch_request->mutable_row_operations()->Clear();
+  replicate->get()->mutable_id()->CopyFrom(mutate_opid);
+  replicate->get()->set_timestamp(clock_->Now().ToUint64());
+  {
+// Modify the existing schema to add an extra row.
+SchemaBuilder builder(schema_);
+ASSERT_OK(builder.AddNullableColumn("string_val_extra", STRING));
+const auto schema = builder.BuildWithoutIds();
+ASSERT_OK(SchemaToPB(schema, batch_request->mutable_schema()));
+
+KuduPartialRow row();
+ASSERT_OK(row.SetInt32("key",

kudu git commit: hms-tool: replace a few copies with const refs

2018-07-23 Thread danburkert

Repository: kudu
Updated Branches:
  refs/heads/master a68c9355b -> ec1e47f41


hms-tool: replace a few copies with const refs

This patch contains no functional changes.

Change-Id: Id2ace8469b82c995f6f528aff4322bf4fed22bd2
Reviewed-on: http://gerrit.cloudera.org:8080/11015
Reviewed-by: Hao Hao 
Tested-by: Kudu Jenkins


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/ec1e47f4
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/ec1e47f4
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/ec1e47f4

Branch: refs/heads/master
Commit: ec1e47f41f25d926609e395d77d0009e210ac17d
Parents: a68c935
Author: Dan Burkert 
Authored: Thu Jul 5 10:06:53 2018 -0700
Committer: Dan Burkert 
Committed: Mon Jul 23 22:54:28 2018 +

--
 src/kudu/tools/tool_action_hms.cc | 36 +++---
 1 file changed, 20 insertions(+), 16 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/ec1e47f4/src/kudu/tools/tool_action_hms.cc
--
diff --git a/src/kudu/tools/tool_action_hms.cc 
b/src/kudu/tools/tool_action_hms.cc
index 70a1aeb..751cf3c 100644
--- a/src/kudu/tools/tool_action_hms.cc
+++ b/src/kudu/tools/tool_action_hms.cc
@@ -97,13 +97,12 @@ unordered_map 
RetrieveTablesMap(vector hms_tab
   return hms_tables_map;
 }
 
-string RenameHiveIncompatibleTable(const string& name) {
-  string table_name(name);
+string RenameHiveIncompatibleTable(const string& table_name) {
   cout << Substitute("Table $0 is not hive compatible.", table_name) << endl;
   cout << "Please input a new table name: ";
-  getline(cin, table_name);
-
-  return table_name;
+  string new_table_name;
+  getline(cin, new_table_name);
+  return new_table_name;
 }
 
 // Only alter the table in Kudu but not in the Hive Metastore.
@@ -285,8 +284,7 @@ Status HmsDowngrade(const RunnerContext& context) {
 
   // 2. Downgrades all Kudu tables to legacy table format.
   for (auto& hms_table : hms_tables) {
-if (hms_table.parameters[HmsClient::kStorageHandlerKey] ==
-HmsClient::kKuduStorageHandler) {
+if (hms_table.parameters[HmsClient::kStorageHandlerKey] == 
HmsClient::kKuduStorageHandler) {
   RETURN_NOT_OK(hms_catalog->DowngradeToLegacyImpalaTable(
   Substitute("$0.$1", hms_table.dbName, hms_table.tableName)));
 }
@@ -338,16 +336,22 @@ Status PrintUnsyncedTables(const string& master_addresses,
   DataTable table({ "TableID", "KuduTableName", "HmsDbName", "HmsTableName",
 "KuduMasterAddresses", "HmsTableMasterAddresses"});
   for (const auto& entry : tables_map) {
-string table_id = entry.first;
-shared_ptr kudu_table = entry.second.first;
-vector hms_tables = entry.second.second;
-string kudu_table_name = kudu_table->name();
+const string& table_id = entry.first;
+const KuduTable& kudu_table = *entry.second.first.get();
+const vector& hms_tables = entry.second.second;
+const string& kudu_table_name = kudu_table.name();
 if (hms_tables.empty()) {
   table.AddRow({ table_id, kudu_table_name, "", "", master_addresses, "" 
});
 } else {
-  for (hive::Table hms_table : hms_tables) {
-table.AddRow({table_id, kudu_table_name, hms_table.dbName, 
hms_table.tableName,
- master_addresses, 
hms_table.parameters[HmsClient::kKuduMasterAddrsKey]});
+  for (const hive::Table& hms_table : hms_tables) {
+table.AddRow({
+table_id,
+kudu_table_name,
+hms_table.dbName,
+hms_table.tableName,
+master_addresses,
+FindOrDie(hms_table.parameters, HmsClient::kKuduMasterAddrsKey),
+});
   }
 }
   }
@@ -362,8 +366,8 @@ Status PrintLegacyTables(const vector& tables, 
ostream& out) {
   DataTable table({ "HmsDbName", "HmsTableName", "KuduTableName",
 "KuduMasterAddresses"});
   for (hive::Table t : tables) {
-string kudu_table_name = t.parameters[HmsClient::kLegacyKuduTableNameKey];
-string master_addresses = t.parameters[HmsClient::kKuduMasterAddrsKey];
+const string& kudu_table_name = 
t.parameters[HmsClient::kLegacyKuduTableNameKey];
+const string& master_addresses = 
t.parameters[HmsClient::kKuduMasterAddrsKey];
 table.AddRow({ t.dbName, t.tableName, kudu_table_name, master_addresses });
   }
   return table.PrintTo(out);

kudu-site git commit: Publish commit(s) from site source repo: e350304 [site] Add new committers (abukor, sailesh)

2018-07-23 Thread abukor

Repository: kudu-site
Updated Branches:
  refs/heads/asf-site 28de58a0b -> 7332ef77a


Publish commit(s) from site source repo:
  e350304 [site] Add new committers (abukor, sailesh)

Site-Repo-Commit: e350304391ebc12018e777b01d4c607d954e2596


Project: http://git-wip-us.apache.org/repos/asf/kudu-site/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu-site/commit/7332ef77
Tree: http://git-wip-us.apache.org/repos/asf/kudu-site/tree/7332ef77
Diff: http://git-wip-us.apache.org/repos/asf/kudu-site/diff/7332ef77

Branch: refs/heads/asf-site
Commit: 7332ef77a3e4f2f58bb90ab0163a5512e088fb15
Parents: 28de58a
Author: Attila Bukor 
Authored: Mon Jul 23 15:01:21 2018 -0700
Committer: Attila Bukor 
Committed: Mon Jul 23 15:01:21 2018 -0700

--
 build-support/kudu_util.pyc | Bin 0 -> 4117 bytes
 committers.html |  10 ++
 feed.xml|   2 +-
 3 files changed, 11 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kudu-site/blob/7332ef77/build-support/kudu_util.pyc
--
diff --git a/build-support/kudu_util.pyc b/build-support/kudu_util.pyc
new file mode 100644
index 000..9a0588a
Binary files /dev/null and b/build-support/kudu_util.pyc differ

http://git-wip-us.apache.org/repos/asf/kudu-site/blob/7332ef77/committers.html
--
diff --git a/committers.html b/committers.html
index 5fb729c..c59a14d 100644
--- a/committers.html
+++ b/committers.html
@@ -112,6 +112,11 @@
   
   
 
+  abukor
+  Attila Bukor
+  PMC
+
+
   adar
   Adar Dembo
   PMC
@@ -192,6 +197,11 @@
   PMC
 
 
+  sailesh
+  Sailesh Mukil
+  PMC
+
+
   stack
   Michael Stack
   PMC

http://git-wip-us.apache.org/repos/asf/kudu-site/blob/7332ef77/feed.xml
--
diff --git a/feed.xml b/feed.xml
index 77c3a12..4fc70b0 100644
--- a/feed.xml
+++ b/feed.xml
@@ -1,4 +1,4 @@
-http://www.w3.org/2005/Atom;>http://jekyllrb.com; 
version="2.5.3">Jekyll2018-07-10T11:50:09-07:00/Instrumentation
 in Apache Kudu2018-07-10T00:00:00-07:002018-07-10T00:00:00-07:00/2018/07/10/instrumentation-in-kudupLast 
week, the a 
href=http://opentracing.io/OpenTracing/a; community 
invited me to
+http://www.w3.org/2005/Atom;>http://jekyllrb.com; 
version="2.5.3">Jekyll2018-07-23T15:01:15-07:00/Instrumentation
 in Apache Kudu2018-07-10T00:00:00-07:002018-07-10T00:00:00-07:00/2018/07/10/instrumentation-in-kudupLast 
week, the a 
href=http://opentracing.io/OpenTracing/a; community 
invited me to
 their monthly Google Hangout meetup to give an informal talk on tracing and
 instrumentation in Apache Kudu./p

kudu git commit: [site] Add new committers (abukor, sailesh)

2018-07-23 Thread abukor

Repository: kudu
Updated Branches:
  refs/heads/gh-pages 35fe952f2 -> e35030439


[site] Add new committers (abukor, sailesh)

Change-Id: Icc5e4bfbd22bb6744b8ddd912e1e4607b0794f41
Reviewed-on: http://gerrit.cloudera.org:8080/11019
Tested-by: Attila Bukor 
Reviewed-by: Dan Burkert 


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/e3503043
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/e3503043
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/e3503043

Branch: refs/heads/gh-pages
Commit: e350304391ebc12018e777b01d4c607d954e2596
Parents: 35fe952
Author: Attila Bukor 
Authored: Mon Jul 23 23:04:16 2018 +0200
Committer: Attila Bukor 
Committed: Mon Jul 23 21:06:36 2018 +

--
 committers.md | 2 ++
 1 file changed, 2 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/e3503043/committers.md
--
diff --git a/committers.md b/committers.md
index 8650e40..15013a8 100644
--- a/committers.md
+++ b/committers.md
@@ -12,6 +12,7 @@ Following is a list of current Kudu committers and members of 
the Kudu PMC.
 
 | Username | Name | Role |
 | --- | --- | --- |
+| abukor | Attila Bukor | PMC |
 | adar | Adar Dembo | PMC |
 | alexey | Alexey Serbin | PMC |
 | awong | Andrew Wong | PMC |
@@ -28,6 +29,7 @@ Following is a list of current Kudu committers and members of 
the Kudu PMC.
 | mattmann | Chris Mattmann | PMC |
 | misty | Misty Stanley-Jones | PMC |
 | mpercy | Mike Percy | PMC |
+| sailesh | Sailesh Mukil | PMC |
 | stack | Michael Stack | PMC |
 | todd | Todd Lipcon | VP, PMC |
 | wdberkeley | William Berkeley | PMC |

[2/2] kudu git commit: [log-test] clean-up on WAL entry test operations

[1/2] kudu git commit: KUDU-2509 fix use-after-free in case of WAL replay error

kudu git commit: hms-tool: replace a few copies with const refs

kudu-site git commit: Publish commit(s) from site source repo: e350304 [site] Add new committers (abukor, sailesh)

kudu git commit: [site] Add new committers (abukor, sailesh)

5 matches

Site Navigation

Mail list logo

Footer information