kudu git commit: error_manager: rename error types
Repository: kudu
Updated Branches:
refs/heads/master 2974f5a50 -> 6ec770928
error_manager: rename error types
The previous ErrorHandlerType enum names weren't very descriptive, and
reflected the expected error handling (e.g. fail a disk, fail a tablet),
rather than the error itself. In preparation of adding a new error
ErrorHandlerType for CFile checksum errors, this patch attempts to
clarify usage of the FsErrorManager.
As I intend on adding another error type, I updated the error manager to
use a map to store its error-handling callbacks, and added a map utility
function to facilitate this.
Change-Id: I377904ee51bb0f7b0f1104d7a4723f74d52e6e3f
Reviewed-on: http://gerrit.cloudera.org:8080/11303
Reviewed-by: Adar Dembo
Tested-by: Kudu Jenkins
Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/6ec77092
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/6ec77092
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/6ec77092
Branch: refs/heads/master
Commit: 6ec77092808e2ddaec2d2cfd8a44b740b43dafa9
Parents: 2974f5a
Author: Andrew Wong
Authored: Thu Aug 23 00:55:10 2018 -0700
Committer: Andrew Wong
Committed: Wed Aug 29 01:52:14 2018 +
--
src/kudu/fs/error_manager-test.cc | 48 ++--
src/kudu/fs/error_manager.cc | 48 ++--
src/kudu/fs/error_manager.h | 50 +-
src/kudu/fs/file_block_manager.cc | 14 -
src/kudu/fs/fs_manager.cc | 2 +-
src/kudu/fs/log_block_manager-test.cc | 2 +-
src/kudu/fs/log_block_manager.cc | 27 +---
src/kudu/gutil/map-util.h | 17 ++
src/kudu/tserver/tablet_server.cc | 4 +--
src/kudu/util/map-util-test.cc| 16 --
10 files changed, 120 insertions(+), 108 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kudu/blob/6ec77092/src/kudu/fs/error_manager-test.cc
--
diff --git a/src/kudu/fs/error_manager-test.cc
b/src/kudu/fs/error_manager-test.cc
index 0c06f0f..33a4866 100644
--- a/src/kudu/fs/error_manager-test.cc
+++ b/src/kudu/fs/error_manager-test.cc
@@ -124,54 +124,54 @@ class FsErrorManagerTest : public KuduTest {
// callbacks) of the error manager.
TEST_F(FsErrorManagerTest, TestBasicRegistration) {
// Before registering anything, there should be all '-1's in test_vec_.
- ASSERT_EQ(-1, FindFirst(ErrorHandlerType::DISK));
- ASSERT_EQ(-1, FindFirst(ErrorHandlerType::TABLET));
+ ASSERT_EQ(-1, FindFirst(ErrorHandlerType::DISK_ERROR));
+ ASSERT_EQ(-1, FindFirst(ErrorHandlerType::NO_AVAILABLE_DISKS));
// Register a callback to update the first '-1' entry in test_vec_ to '0'
// after waiting a random amount of time.
- em()->SetErrorNotificationCb(ErrorHandlerType::DISK,
+ em()->SetErrorNotificationCb(ErrorHandlerType::DISK_ERROR,
Bind(::SleepAndWriteFirstEmptyCb,
- Unretained(this), ErrorHandlerType::DISK));
- em()->RunErrorNotificationCb(ErrorHandlerType::DISK, "");
- ASSERT_EQ(0, FindFirst(ErrorHandlerType::DISK));
+ Unretained(this), ErrorHandlerType::DISK_ERROR));
+ em()->RunErrorNotificationCb(ErrorHandlerType::DISK_ERROR, "");
+ ASSERT_EQ(0, FindFirst(ErrorHandlerType::DISK_ERROR));
// Running callbacks that haven't been registered should do nothing.
- em()->RunErrorNotificationCb(ErrorHandlerType::TABLET, "");
- ASSERT_EQ(0, FindFirst(ErrorHandlerType::DISK));
- ASSERT_EQ(-1, FindFirst(ErrorHandlerType::TABLET));
+ em()->RunErrorNotificationCb(ErrorHandlerType::NO_AVAILABLE_DISKS, "");
+ ASSERT_EQ(0, FindFirst(ErrorHandlerType::DISK_ERROR));
+ ASSERT_EQ(-1, FindFirst(ErrorHandlerType::NO_AVAILABLE_DISKS));
// Now register another callback.
- em()->SetErrorNotificationCb(ErrorHandlerType::TABLET,
+ em()->SetErrorNotificationCb(ErrorHandlerType::NO_AVAILABLE_DISKS,
Bind(::SleepAndWriteFirstEmptyCb,
- Unretained(this), ErrorHandlerType::TABLET));
- em()->RunErrorNotificationCb(ErrorHandlerType::TABLET, "");
- ASSERT_EQ(1, FindFirst(ErrorHandlerType::TABLET));
+ Unretained(this), ErrorHandlerType::NO_AVAILABLE_DISKS));
+ em()->RunErrorNotificationCb(ErrorHandlerType::NO_AVAILABLE_DISKS, "");
+ ASSERT_EQ(1, FindFirst(ErrorHandlerType::NO_AVAILABLE_DISKS));
// Now unregister one of the callbacks. This should not affect the other.
- em()->UnsetErrorNotificationCb(ErrorHandlerType::DISK);
- em()->RunErrorNotificationCb(ErrorHandlerType::DISK, "");
- em()->RunErrorNotificationCb(ErrorHandlerType::TABLET, "");
+ em()->UnsetErrorNotificationCb(ErrorHandlerType::DISK_ERROR);
+ em()->RunErrorNotificationCb(ErrorHandlerType::DISK_ERROR, "");
+ em()->RunErrorNotificationCb(ErrorHandlerType::NO_AVAILABLE_DISKS, "");
[2/3] kudu git commit: KUDU-2469 pt 1: add an IOContext
http://git-wip-us.apache.org/repos/asf/kudu/blob/2974f5a5/src/kudu/tablet/deltamemstore.h
--
diff --git a/src/kudu/tablet/deltamemstore.h b/src/kudu/tablet/deltamemstore.h
index b637e00..253a2c9 100644
--- a/src/kudu/tablet/deltamemstore.h
+++ b/src/kudu/tablet/deltamemstore.h
@@ -57,6 +57,10 @@ namespace consensus {
class OpId;
}
+namespace fs {
+struct IOContext;
+}
+
namespace tablet {
class DeltaFileWriter;
@@ -78,7 +82,7 @@ class DeltaMemStore : public DeltaStore,
std::shared_ptr parent_tracker,
std::shared_ptr* dms);
- virtual Status Init() OVERRIDE;
+ virtual Status Init(const fs::IOContext* io_context) OVERRIDE;
virtual bool Initted() OVERRIDE {
return true;
@@ -122,7 +126,8 @@ class DeltaMemStore : public DeltaStore,
virtual Status NewDeltaIterator(const RowIteratorOptions& opts,
DeltaIterator** iterator) const OVERRIDE;
- virtual Status CheckRowDeleted(rowid_t row_idx, bool *deleted) const
OVERRIDE;
+ virtual Status CheckRowDeleted(rowid_t row_idx, const fs::IOContext*
io_context,
+ bool* deleted) const OVERRIDE;
virtual uint64_t EstimateSize() const OVERRIDE {
return arena_->memory_footprint();
http://git-wip-us.apache.org/repos/asf/kudu/blob/2974f5a5/src/kudu/tablet/diskrowset-test-base.h
--
diff --git a/src/kudu/tablet/diskrowset-test-base.h
b/src/kudu/tablet/diskrowset-test-base.h
index 9742169..ec3f4b3 100644
--- a/src/kudu/tablet/diskrowset-test-base.h
+++ b/src/kudu/tablet/diskrowset-test-base.h
@@ -186,7 +186,7 @@ class TestRowSet : public KuduRowSetTest {
ProbeStats stats;
ScopedTransaction tx(_, clock_->Now());
tx.StartApplying();
-Status s = rs->MutateRow(tx.timestamp(), probe, mutation, op_id_, ,
result);
+Status s = rs->MutateRow(tx.timestamp(), probe, mutation, op_id_, nullptr,
, result);
tx.Commit();
return s;
}
@@ -196,7 +196,7 @@ class TestRowSet : public KuduRowSetTest {
BuildRowKey(, row_idx);
RowSetKeyProbe probe(rb.row());
ProbeStats stats;
-return rs.CheckRowPresent(probe, present, );
+return rs.CheckRowPresent(probe, nullptr, present, );
}
// Verify the contents of the given rowset.
@@ -327,6 +327,7 @@ class TestRowSet : public KuduRowSetTest {
return DiskRowSet::Open(rowset_meta_,
new log::LogAnchorRegistry(),
TabletMemTrackers(),
+nullptr,
rowset);
}
http://git-wip-us.apache.org/repos/asf/kudu/blob/2974f5a5/src/kudu/tablet/diskrowset-test.cc
--
diff --git a/src/kudu/tablet/diskrowset-test.cc
b/src/kudu/tablet/diskrowset-test.cc
index ce0fb7d..7e89122 100644
--- a/src/kudu/tablet/diskrowset-test.cc
+++ b/src/kudu/tablet/diskrowset-test.cc
@@ -121,7 +121,7 @@ TEST_F(TestRowSet, TestRowSetRoundTrip) {
rb.AddString(Slice("h"));
RowSetKeyProbe probe(rb.row());
bool present;
-ASSERT_OK(rs->CheckRowPresent(probe, , ));
+ASSERT_OK(rs->CheckRowPresent(probe, nullptr, , ));
ASSERT_FALSE(present);
}
@@ -131,7 +131,7 @@ TEST_F(TestRowSet, TestRowSetRoundTrip) {
rb.AddString(Slice("z"));
RowSetKeyProbe probe(rb.row());
bool present;
-ASSERT_OK(rs->CheckRowPresent(probe, , ));
+ASSERT_OK(rs->CheckRowPresent(probe, nullptr, , ));
ASSERT_FALSE(present);
}
@@ -142,7 +142,7 @@ TEST_F(TestRowSet, TestRowSetRoundTrip) {
rb.AddString(Slice("hello 49x"));
RowSetKeyProbe probe(rb.row());
bool present;
-ASSERT_OK(rs->CheckRowPresent(probe, , ));
+ASSERT_OK(rs->CheckRowPresent(probe, nullptr, , ));
ASSERT_FALSE(present);
}
@@ -154,7 +154,7 @@ TEST_F(TestRowSet, TestRowSetRoundTrip) {
rb.AddString(Slice(buf));
RowSetKeyProbe probe(rb.row());
bool present;
-ASSERT_OK(rs->CheckRowPresent(probe, , ));
+ASSERT_OK(rs->CheckRowPresent(probe, nullptr, , ));
ASSERT_TRUE(present);
}
}
@@ -188,7 +188,7 @@ TEST_F(TestRowSet, TestRowSetUpdate) {
OperationResultPB result;
ProbeStats stats;
- Status s = rs->MutateRow(timestamp, probe, enc.as_changelist(), op_id_,
, );
+ Status s = rs->MutateRow(timestamp, probe, enc.as_changelist(), op_id_,
nullptr, , );
ASSERT_TRUE(s.IsNotFound());
ASSERT_EQ(0, result.mutated_stores_size());
@@ -219,7 +219,7 @@ TEST_F(TestRowSet, TestErrorDuringUpdate) {
// The mutation should result in an IOError.
OperationResultPB result;
ProbeStats stats;
- Status s = rs->MutateRow(timestamp, probe, enc.as_changelist(), op_id_,
, );
+ Status s = rs->MutateRow(timestamp, probe, enc.as_changelist(), op_id_,
nullptr, , );
LOG(INFO) << s.ToString();
[1/3] kudu git commit: KUDU-2469 pt 1: add an IOContext
Repository: kudu
Updated Branches:
refs/heads/master 846eb7f97 -> 2974f5a50
http://git-wip-us.apache.org/repos/asf/kudu/blob/2974f5a5/src/kudu/tablet/tablet_bootstrap.cc
--
diff --git a/src/kudu/tablet/tablet_bootstrap.cc
b/src/kudu/tablet/tablet_bootstrap.cc
index 670d6ee..e70372a 100644
--- a/src/kudu/tablet/tablet_bootstrap.cc
+++ b/src/kudu/tablet/tablet_bootstrap.cc
@@ -53,6 +53,7 @@
#include "kudu/fs/data_dirs.h"
#include "kudu/fs/fs.pb.h"
#include "kudu/fs/fs_manager.h"
+#include "kudu/fs/io_context.h"
#include "kudu/gutil/bind.h"
#include "kudu/gutil/gscoped_ptr.h"
#include "kudu/gutil/macros.h"
@@ -115,6 +116,7 @@ using kudu::consensus::OperationType_Name;
using kudu::consensus::RaftConfigPB;
using kudu::consensus::ReplicateMsg;
using kudu::consensus::WRITE_OP;
+using kudu::fs::IOContext;
using kudu::log::Log;
using kudu::log::LogAnchorRegistry;
using kudu::log::LogEntryPB;
@@ -254,28 +256,29 @@ class TabletBootstrap {
// Plays the log segments into the tablet being built.
// The process of playing the segments generates a new log that can be
continued
// later on when then tablet is rebuilt and starts accepting writes from
clients.
- Status PlaySegments(ConsensusBootstrapInfo* results);
+ Status PlaySegments(const IOContext* io_context, ConsensusBootstrapInfo*
consensus_info);
// Append the given commit message to the log.
// Does not support writing a TxResult.
Status AppendCommitMsg(const CommitMsg& commit_msg);
- Status PlayWriteRequest(ReplicateMsg* replicate_msg,
+ Status PlayWriteRequest(const IOContext* io_context, ReplicateMsg*
replicate_msg,
const CommitMsg& commit_msg);
- Status PlayAlterSchemaRequest(ReplicateMsg* replicate_msg,
+ Status PlayAlterSchemaRequest(const IOContext* io_context, ReplicateMsg*
replicate_msg,
const CommitMsg& commit_msg);
- Status PlayChangeConfigRequest(ReplicateMsg* replicate_msg,
+ Status PlayChangeConfigRequest(const IOContext* io_context, ReplicateMsg*
replicate_msg,
const CommitMsg& commit_msg);
- Status PlayNoOpRequest(ReplicateMsg* replicate_msg,
+ Status PlayNoOpRequest(const IOContext* io_context, ReplicateMsg*
replicate_msg,
const CommitMsg& commit_msg);
// Plays operations, skipping those that have already been flushed or have
previously failed.
// See ApplyRowOperations() for more details on how the decision of whether
an operation
// is applied or skipped is made.
- Status PlayRowOperations(WriteTransactionState* tx_state,
+ Status PlayRowOperations(const IOContext* io_context,
+ WriteTransactionState* tx_state,
const TxResultPB& orig_result,
TxResultPB* new_result);
@@ -294,7 +297,8 @@ class TabletBootstrap {
// - if it was previously failed, mark as failed
// - if it previously succeeded but was flushed, skip it.
// - otherwise, re-apply to the tablet being bootstrapped.
- Status ApplyOperations(WriteTransactionState* tx_state,
+ Status ApplyOperations(const IOContext* io_context,
+ WriteTransactionState* tx_state,
const TxResultPB& orig_result,
TxResultPB* new_result);
@@ -336,7 +340,8 @@ class TabletBootstrap {
void DumpReplayStateToLog(const ReplayState& state);
- Status HandleEntry(ReplayState* state,
+ Status HandleEntry(const IOContext* io_context,
+ ReplayState* state,
unique_ptr entry,
string* entry_debug_info);
@@ -344,12 +349,13 @@ class TabletBootstrap {
Status HandleReplicateMessage(ReplayState* state,
unique_ptr entry,
string* entry_debug_info);
- Status HandleCommitMessage(ReplayState* state,
+ Status HandleCommitMessage(const IOContext* io_context, ReplayState* state,
unique_ptr entry,
string* entry_debug_info);
- Status ApplyCommitMessage(ReplayState* state, LogEntryPB* entry);
- Status HandleEntryPair(LogEntryPB* replicate_entry, LogEntryPB*
commit_entry);
+ Status ApplyCommitMessage(const IOContext* io_context, ReplayState* state,
LogEntryPB* entry);
+ Status HandleEntryPair(const IOContext* io_context, LogEntryPB*
replicate_entry,
+ LogEntryPB* commit_entry);
// Checks that an orphaned commit message is actually irrelevant, i.e that
none
// of the data stores it refers to are live.
@@ -601,7 +607,8 @@ Status TabletBootstrap::RunBootstrap(shared_ptr*
rebuilt_tablet,
tablet_id));
}
- RETURN_NOT_OK_PREPEND(PlaySegments(consensus_info), "Failed log replay.
Reason");
+ IOContext io_context({
[3/3] kudu git commit: KUDU-2469 pt 1: add an IOContext
KUDU-2469 pt 1: add an IOContext
This patch introduces the IOContext class that can be used to pass state
from the Tablet layer to the CFile layer. The top-level caller (whether
it is a Tablet method, an iterator, etc.) will own the IOContext, and
pass a pointer to it to lower-level classes.
I intend on using this to pass the tablet id down to the CFileReaders
for error handling. As such, only codepaths that lead to CFileReaders
are plumbed. Currently the IOContext only contains a tablet id, though
in the future it could be used for more sophisticated things like
resource management, QoS for IO, etc.
Change-Id: I645e10a2fda1f2564326b7052e6a0debc5ad738c
Reviewed-on: http://gerrit.cloudera.org:8080/11248
Reviewed-by: Adar Dembo
Tested-by: Kudu Jenkins
Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/2974f5a5
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/2974f5a5
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/2974f5a5
Branch: refs/heads/master
Commit: 2974f5a50c95862960d09f3f18831ca259d921b2
Parents: 846eb7f
Author: Andrew Wong
Authored: Thu Aug 23 00:17:26 2018 -0700
Committer: Andrew Wong
Committed: Wed Aug 29 00:40:15 2018 +
--
src/kudu/cfile/bloomfile-test-base.h| 2 +-
src/kudu/cfile/bloomfile-test.cc| 8 +-
src/kudu/cfile/bloomfile.cc | 16 +--
src/kudu/cfile/bloomfile.h | 8 +-
src/kudu/cfile/cfile-test-base.h| 2 +-
src/kudu/cfile/cfile-test.cc| 10 +-
src/kudu/cfile/cfile_reader.cc | 26 +++--
src/kudu/cfile/cfile_reader.h | 27 +++--
src/kudu/cfile/cfile_util.h | 9 ++
src/kudu/fs/io_context.h| 32 ++
src/kudu/tablet/cfile_set-test.cc | 16 +--
src/kudu/tablet/cfile_set.cc| 60 ++-
src/kudu/tablet/cfile_set.h | 36 ---
src/kudu/tablet/compaction-test.cc | 31 +++---
src/kudu/tablet/compaction.cc | 17 +--
src/kudu/tablet/compaction.h| 8 +-
src/kudu/tablet/delta_compaction.cc | 17 +--
src/kudu/tablet/delta_compaction.h | 10 +-
src/kudu/tablet/delta_store.h | 9 +-
src/kudu/tablet/delta_tracker.cc| 74 +++--
src/kudu/tablet/delta_tracker.h | 41
src/kudu/tablet/deltafile-test.cc | 4 +-
src/kudu/tablet/deltafile.cc| 24 +++--
src/kudu/tablet/deltafile.h | 9 +-
src/kudu/tablet/deltamemstore-test.cc | 2 +-
src/kudu/tablet/deltamemstore.cc| 7 +-
src/kudu/tablet/deltamemstore.h | 9 +-
src/kudu/tablet/diskrowset-test-base.h | 5 +-
src/kudu/tablet/diskrowset-test.cc | 51 -
src/kudu/tablet/diskrowset.cc | 70 -
src/kudu/tablet/diskrowset.h| 24 +++--
src/kudu/tablet/memrowset-test.cc | 4 +-
src/kudu/tablet/memrowset.cc| 7 +-
src/kudu/tablet/memrowset.h | 23 +++--
src/kudu/tablet/mock-rowsets.h | 42 +---
src/kudu/tablet/mt-diskrowset-test.cc | 2 +-
.../tablet/mt-rowset_delta_compaction-test.cc | 4 +-
src/kudu/tablet/rowset.cc | 23 +++--
src/kudu/tablet/rowset.h| 38 +--
src/kudu/tablet/tablet.cc | 103 ---
src/kudu/tablet/tablet.h| 30 --
src/kudu/tablet/tablet_bootstrap.cc | 94 ++---
src/kudu/tablet/tablet_history_gc-test.cc | 2 +-
src/kudu/tools/tool_action_fs.cc| 2 +-
src/kudu/tools/tool_action_local_replica.cc | 1 +
45 files changed, 658 insertions(+), 381 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kudu/blob/2974f5a5/src/kudu/cfile/bloomfile-test-base.h
--
diff --git a/src/kudu/cfile/bloomfile-test-base.h
b/src/kudu/cfile/bloomfile-test-base.h
index a702d32..74251b1 100644
--- a/src/kudu/cfile/bloomfile-test-base.h
+++ b/src/kudu/cfile/bloomfile-test-base.h
@@ -112,7 +112,7 @@ class BloomFileTestBase : public KuduTest {
Slice s(reinterpret_cast(), sizeof(key));
bool present;
-CHECK_OK(bfr_->CheckKeyPresent(BloomKeyProbe(s), ));
+CHECK_OK(bfr_->CheckKeyPresent(BloomKeyProbe(s), nullptr, ));
if (present) count_present++;
}
}
http://git-wip-us.apache.org/repos/asf/kudu/blob/2974f5a5/src/kudu/cfile/bloomfile-test.cc
[2/2] kudu git commit: KUDU-428: Sentry integration scaffolding
KUDU-428: Sentry integration scaffolding
In preparation for KUDU-428 (Sentry integration), this commit introduces
some basic module and build-system scaffolding, including a new 'sentry'
module, as well as the Sentry service thrift definition files. This is
heavily based on the HMS integration equivalents.
Change-Id: I51e68299b97f74f6844bfa9f8aba4c0bd4246c11
Reviewed-on: http://gerrit.cloudera.org:8080/11292
Reviewed-by: Adar Dembo
Tested-by: Kudu Jenkins
Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/846eb7f9
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/846eb7f9
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/846eb7f9
Branch: refs/heads/master
Commit: 846eb7f9705812515fadffe1eacb69bf60ad5a9b
Parents: 81ab99b
Author: Dan Burkert
Authored: Fri Aug 17 12:05:53 2018 -0700
Committer: Dan Burkert
Committed: Tue Aug 28 23:34:17 2018 +
--
CMakeLists.txt | 3 +-
cmake_modules/FindThrift.cmake | 14 +-
src/kudu/hms/CMakeLists.txt | 16 +-
src/kudu/mini-cluster/CMakeLists.txt | 2 +-
src/kudu/sentry/CMakeLists.txt | 67
src/kudu/sentry/mini_sentry.cc | 23 ++
src/kudu/sentry/mini_sentry.h| 28 ++
src/kudu/sentry/sentry_client-test.cc| 39 +++
src/kudu/sentry/sentry_client.cc | 23 ++
src/kudu/sentry/sentry_client.h | 28 ++
src/kudu/sentry/sentry_common_service.thrift | 50 +++
src/kudu/sentry/sentry_policy_service.thrift | 370 ++
src/kudu/sentry/thrift_operators.cc | 62
13 files changed, 717 insertions(+), 8 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kudu/blob/846eb7f9/CMakeLists.txt
--
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 7029a4a..0cd6bc1 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -1374,7 +1374,7 @@ endif (UNIX)
if (UNIX)
add_custom_target(iwyu-generated-headers
-DEPENDS pb-gen krpc-gen hms_thrift)
+DEPENDS pb-gen krpc-gen hms_thrift sentry_thrift)
add_custom_target(iwyu ${BUILD_SUPPORT_DIR}/iwyu.py --from-git
DEPENDS iwyu-generated-headers)
add_custom_target(iwyu-fix ${BUILD_SUPPORT_DIR}/iwyu.py --fix --from-git
@@ -1474,6 +1474,7 @@ add_subdirectory(src/kudu/master)
add_subdirectory(src/kudu/mini-cluster)
add_subdirectory(src/kudu/rpc)
add_subdirectory(src/kudu/security)
+add_subdirectory(src/kudu/sentry)
add_subdirectory(src/kudu/server)
add_subdirectory(src/kudu/tablet)
add_subdirectory(src/kudu/tools)
http://git-wip-us.apache.org/repos/asf/kudu/blob/846eb7f9/cmake_modules/FindThrift.cmake
--
diff --git a/cmake_modules/FindThrift.cmake b/cmake_modules/FindThrift.cmake
index 834bbfd..3b9b217 100644
--- a/cmake_modules/FindThrift.cmake
+++ b/cmake_modules/FindThrift.cmake
@@ -51,6 +51,8 @@
# custom targets; if SRCS/HDRS need to be used in multiple
# libraries, those libraries should depend on these targets
# in order to "serialize" the thrift invocations
+# FB303 = Option which determines if the Thrift definitions depend on the
+# FB303 support library.
#
function(THRIFT_GENERATE_CPP SRCS HDRS TGTS)
@@ -59,7 +61,7 @@ function(THRIFT_GENERATE_CPP SRCS HDRS TGTS)
return()
endif(NOT ARGN)
- set(options)
+ set(options FB303)
set(one_value_args SOURCE_ROOT BINARY_ROOT)
set(multi_value_args EXTRA_THRIFT_PATHS THRIFT_FILES)
cmake_parse_arguments(ARG "${options}" "${one_value_args}"
"${multi_value_args}" ${ARGN})
@@ -102,9 +104,13 @@ function(THRIFT_GENERATE_CPP SRCS HDRS TGTS)
list(APPEND THRIFT_CC_OUT "${ARG_BINARY_ROOT}/${SERVICE}.cpp")
endforeach()
-# TODO(dan): Add the fb303 files manually. This is a complete hack.
-list(APPEND ${SRCS} "${THRIFT_CC_OUT}" "fb303_types.cpp"
"fb303_constants.cpp" "FacebookService.cpp")
-list(APPEND ${HDRS} "${THRIFT_H_OUT}" "fb303_types.h" "fb303_constants.h"
"FacebookService.h")
+list(APPEND ${SRCS} "${THRIFT_CC_OUT}")
+list(APPEND ${HDRS} "${THRIFT_H_OUT}")
+
+if(ARG_FB303)
+ list(APPEND ${SRCS} fb303_types.cpp fb303_constants.cpp
FacebookService.cpp)
+ list(APPEND ${HDRS} fb303_types.h fb303_constants.h FacebookService.h)
+endif()
add_custom_command(
OUTPUT ${THRIFT_CC_OUT} ${THRIFT_H_OUT}
http://git-wip-us.apache.org/repos/asf/kudu/blob/846eb7f9/src/kudu/hms/CMakeLists.txt
--
diff --git a/src/kudu/hms/CMakeLists.txt
[2/2] kudu git commit: build: different libunwind workaround for 0dc19bfb3
build: different libunwind workaround for 0dc19bfb3
Rather than add libunwind to the gutil test dependency list twice (once
explicitly, and once via KUDU_BASE_LIBS), let's express it as a dependency of
glog. That'll cause cmake to adjust the link order based on the new dependency
graph, and place libunwind early enough that glog can resolve symbols in it.
I tested this by building the two gutil tests in release mode. After removing
the workaround from gutil/CMakeLists.txt, they failed to build. With the new
dependency information, they build successfully.
Change-Id: I18325dbbddf82f8bad7d7b9bf759487811396ff3
Reviewed-on: http://gerrit.cloudera.org:8080/11343
Tested-by: Kudu Jenkins
Reviewed-by: Alexey Serbin
Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/d55df3c6
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/d55df3c6
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/d55df3c6
Branch: refs/heads/master
Commit: d55df3c6e6f3c6540d5eec9b42f3138b6ebe51c5
Parents: 5d69deb
Author: Adar Dembo
Authored: Tue Aug 28 12:15:42 2018 -0700
Committer: Adar Dembo
Committed: Tue Aug 28 21:23:13 2018 +
--
CMakeLists.txt| 12 +---
src/kudu/gutil/CMakeLists.txt | 1 -
2 files changed, 9 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kudu/blob/d55df3c6/CMakeLists.txt
--
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 11b47b6..7029a4a 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -984,15 +984,21 @@ ADD_THIRDPARTY_LIB(gssapi_krb5
SHARED_LIB "${GSSAPI_SHARED_LIB}"
DEPS ${GSSAPI_LIB_DEPS})
-## GLog
+## GLog (depends on libunwind)
find_package(GLog REQUIRED)
include_directories(SYSTEM ${GLOG_INCLUDE_DIR})
+set(GLOG_DEPS)
+if (NOT APPLE)
+ set(GLOG_DEPS unwind)
+endif()
ADD_THIRDPARTY_LIB(glog
STATIC_LIB "${GLOG_STATIC_LIB}"
- SHARED_LIB "${GLOG_SHARED_LIB}")
+ SHARED_LIB "${GLOG_SHARED_LIB}"
+ DEPS "${GLOG_DEPS}")
list(APPEND KUDU_BASE_LIBS glog)
-## libunwind (dependent of glog)
+## libunwind
+##
## Doesn't build on OSX.
if (NOT APPLE)
find_package(LibUnwind REQUIRED)
http://git-wip-us.apache.org/repos/asf/kudu/blob/d55df3c6/src/kudu/gutil/CMakeLists.txt
--
diff --git a/src/kudu/gutil/CMakeLists.txt b/src/kudu/gutil/CMakeLists.txt
index 787e6db..3d487eb 100644
--- a/src/kudu/gutil/CMakeLists.txt
+++ b/src/kudu/gutil/CMakeLists.txt
@@ -59,7 +59,6 @@ set(GUTIL_LIBS
if (NOT APPLE)
set(GUTIL_LIBS
${GUTIL_LIBS}
-unwind
rt) # clock_gettime() requires -lrt
endif()
[1/2] kudu git commit: build-support: option to retry all failed tests
Repository: kudu
Updated Branches:
refs/heads/master 3bb5f56cf -> d55df3c6e
build-support: option to retry all failed tests
Currently, users can opt to retry flaky tests as reported by the
user-specified test server. The test server's flaky test list may not
accurately reflect what tests are flaky in all environments. In
environments where there are flaky tests that are under-represented by
the test server, it would still be nice to be resilient to flakies. As
such, this patch adds an option to retry all failed tests.
Here's a run of a non-flaky test into which I added a FATAL log.
http://dist-test.cloudera.org/job?job_id=awong.1535433877.28172
Change-Id: I24aea0b9e7a1c2c66bc5feffcb454ff01cdca6fd
Reviewed-on: http://gerrit.cloudera.org:8080/11342
Tested-by: Kudu Jenkins
Reviewed-by: Grant Henke
Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/5d69deb3
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/5d69deb3
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/5d69deb3
Branch: refs/heads/master
Commit: 5d69deb36925113796cd69f51061b8396b0174fc
Parents: 3bb5f56
Author: Andrew Wong
Authored: Mon Aug 27 19:03:08 2018 -0700
Committer: Andrew Wong
Committed: Tue Aug 28 15:39:46 2018 +
--
build-support/dist_test.py | 9 -
build-support/run-test.sh | 35 ---
2 files changed, 28 insertions(+), 16 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kudu/blob/5d69deb3/build-support/dist_test.py
--
diff --git a/build-support/dist_test.py b/build-support/dist_test.py
index ad2bcf1..c1176fe 100755
--- a/build-support/dist_test.py
+++ b/build-support/dist_test.py
@@ -58,6 +58,11 @@ MAX_TASKS_PER_JOB=1
# of retries, so we have to subtract 1.
FLAKY_TEST_RETRIES = int(os.environ.get('KUDU_FLAKY_TEST_ATTEMPTS', 1)) - 1
+# Whether to retry all failed C++ tests, rather than just known flaky tests.
+# Since Java flaky tests are not reported by the test server, Java tests are
+# always retried, regardless of this value.
+RETRY_ALL_TESTS = int(os.environ.get('KUDU_RETRY_ALL_FAILED_TESTS', 0))
+
# Flags to include when running Gradle tasks
GRADLE_FLAGS = os.environ.get('EXTRA_GRADLE_FLAGS', "")
@@ -473,9 +478,11 @@ def run_tests(parser, options):
create_archive_input(staging, execution,
collect_tmpdir=options.collect_tmpdir)
run_isolate(staging)
+ retry_all = RETRY_ALL_TESTS > 0
create_task_json(staging,
flaky_test_set=get_flakies(),
- replicate_tasks=options.num_instances)
+ replicate_tasks=options.num_instances,
+ retry_all_tests=retry_all)
submit_tasks(staging, options)
def add_run_subparser(subparsers):
http://git-wip-us.apache.org/repos/asf/kudu/blob/5d69deb3/build-support/run-test.sh
--
diff --git a/build-support/run-test.sh b/build-support/run-test.sh
index 541d6f7..e8cc996 100755
--- a/build-support/run-test.sh
+++ b/build-support/run-test.sh
@@ -23,11 +23,12 @@
# If KUDU_COMPRESS_TEST_OUTPUT is non-empty, then the logs will be
# gzip-compressed while they are written.
#
-# If KUDU_FLAKY_TEST_ATTEMPTS is non-zero, and the test being run matches
-# one of the lines in the file KUDU_FLAKY_TEST_LIST, then the test will
-# be retried on failure up to the specified number of times. This can be
-# used in the gerrit workflow to prevent annoying false -1s caused by
-# tests that are known to be flaky in master.
+# If KUDU_FLAKY_TEST_ATTEMPTS is non-zero, and either the test being run
+# matches one of the lines in the file KUDU_FLAKY_TEST_LIST or
+# KUDU_RETRY_ALL_FAILED_TESTS is non-zero, then the test will be retried on
+# failure up to the specified number of times. This can be used in the gerrit
+# workflow to prevent annoying false -1s caused by tests that are known to be
+# flaky in master.
#
# If KUDU_REPORT_TEST_RESULTS is non-zero, then tests are reported to the
# central test server.
@@ -70,23 +71,27 @@ else
TEST_NAME=${SHORT_TEST_NAME}
fi
-# Determine whether the test is a known flaky by comparing against the
user-specified
-# list.
+# Determine whether the user has chosen to retry all failed tests, or whether
+# the test is a known flaky by comparing against the user-specified list.
TEST_EXECUTION_ATTEMPTS=1
-if [ -n "$KUDU_FLAKY_TEST_LIST" ]; then
+if [ "$KUDU_RETRY_ALL_FAILED_TESTS" -gt 0 ]; then
+ echo "Will retry on failure"
+ TEST_IS_RETRYABLE=1
+elif [ -n "KUDU_FLAKY_TEST_LIST" ]; then
if [ -f "$KUDU_FLAKY_TEST_LIST" ]; then
-IS_KNOWN_FLAKY=$(grep --count --line-regexp "$SHORT_TEST_NAME"
"$KUDU_FLAKY_TEST_LIST")
+TEST_IS_RETRYABLE=$(grep --count
