kudu git commit: Update website with doc improvements from master
Repository: kudu Updated Branches: refs/heads/gh-pages 4cdc8268c -> 8e3bce7ff Update website with doc improvements from master I generated updated docs and manually filtered out any changes that are not relevant for the current or past releases. Change-Id: Ic846d497793bd16022cad7f0b185979bd68639a6 Reviewed-on: http://gerrit.cloudera.org:8080/10730 Reviewed-by: Will Berkeley Reviewed-by: Alex Rodoni Reviewed-by: Mike Percy Tested-by: Grant Henke Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/8e3bce7f Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/8e3bce7f Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/8e3bce7f Branch: refs/heads/gh-pages Commit: 8e3bce7ffcc8dd0dea4fabfa6981b6bed5c72dd4 Parents: 4cdc826 Author: Grant Henke Authored: Fri Jun 15 09:16:59 2018 -0500 Committer: Grant Henke Committed: Fri Jun 15 21:37:34 2018 + -- docs/administration.html| 193 +-- docs/background_tasks.html | 6 +- docs/command_line_tools.html| 6 +- docs/command_line_tools_reference.html | 6 +- docs/configuration.html | 12 +- docs/configuration_reference.html | 6 +- docs/configuration_reference_unsupported.html | 6 +- docs/contributing.html | 40 ++- docs/developing.html| 55 +++-- docs/export_control.html| 6 +- docs/index.html | 6 +- docs/installation.html | 35 ++- docs/known_issues.html | 17 +- docs/kudu-master_configuration_reference.html | 6 +- ...ter_configuration_reference_unsupported.html | 6 +- docs/kudu-tserver_configuration_reference.html | 6 +- ...ver_configuration_reference_unsupported.html | 6 +- docs/kudu_impala_integration.html | 41 +++- docs/quickstart.html| 6 +- docs/scaling_guide.html | 16 +- docs/schema_design.html | 6 +- docs/security.html | 6 +- docs/transaction_semantics.html | 6 +- docs/troubleshooting.html | 246 +++ 24 files changed, 612 insertions(+), 133 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kudu/blob/8e3bce7f/docs/administration.html -- diff --git a/docs/administration.html b/docs/administration.html index 9184d79..86b445d 100644 --- a/docs/administration.html +++ b/docs/administration.html @@ -2,7 +2,7 @@ title: Apache Kudu Administration layout: default active_nav: docs -last_updated: 'Last updated 2018-03-17 09:37:29 CDT' +last_updated: 'Last updated 2018-06-14 08:17:56 PDT' ---
[2/2] kudu git commit: KUDU-2191: allow metadata upgrade tool to run with HMS integration
KUDU-2191: allow metadata upgrade tool to run with HMS integration
Previously the metadata upgrade tool requires the HMS integration
feature being disabled. This commit updates the tool to only alter
tables in Kudu but not in the HMS during upgrade, so that the
restriction no longer applies.
Change-Id: I51d4cabeb1a9defc51f4e307a116419da5588f2d
Reviewed-on: http://gerrit.cloudera.org:8080/10582
Tested-by: Kudu Jenkins
Reviewed-by: Dan Burkert
Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/f2089195
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/f2089195
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/f2089195
Branch: refs/heads/master
Commit: f2089195d635ea73e7268a9b46db2feac2eeed2d
Parents: e3474f4
Author: hahao
Authored: Wed Jun 6 23:58:49 2018 -0700
Committer: Hao Hao
Committed: Fri Jun 15 20:07:45 2018 +
--
src/kudu/client/client.h | 16 +++-
src/kudu/tools/kudu-tool-test.cc | 5 +
src/kudu/tools/tool_action_hms.cc | 21 -
3 files changed, 32 insertions(+), 10 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kudu/blob/f2089195/src/kudu/client/client.h
--
diff --git a/src/kudu/client/client.h b/src/kudu/client/client.h
index 3404b9a..f03c0e3 100644
--- a/src/kudu/client/client.h
+++ b/src/kudu/client/client.h
@@ -37,6 +37,7 @@
#include "kudu/client/shared_ptr.h" // IWYU pragma: keep
#ifdef KUDU_HEADERS_NO_STUBS
#include
+
#include "kudu/gutil/macros.h"
#include "kudu/gutil/port.h"
#else
@@ -55,13 +56,19 @@ class MonoDelta;
class PartitionSchema;
class SecurityUnknownTskTest;
+namespace client {
+class KuduClient;
+}
+
namespace tools {
class LeaderMasterProxy;
+
+Status AlterKuduTable(const client::sp::shared_ptr&
kudu_client,
+ const std::string& name, const std::string& new_name);
} // namespace tools
namespace client {
-class KuduClient;
class KuduDelete;
class KuduInsert;
class KuduLoggingCallback;
@@ -1181,8 +1188,14 @@ class KUDU_EXPORT KuduTableAlterer {
private:
class KUDU_NO_EXPORT Data;
+
friend class KuduClient;
+ friend Status tools::AlterKuduTable(
+ const client::sp::shared_ptr& kudu_client,
+ const std::string& name,
+ const std::string& new_name);
+
FRIEND_TEST(kudu::MasterHmsTest, TestAlterTable);
KuduTableAlterer(KuduClient* client,
@@ -2391,6 +2404,7 @@ class KUDU_EXPORT KuduPartitioner {
Status PartitionRow(const KuduPartialRow& row, int* partition);
private:
class KUDU_NO_EXPORT Data;
+
friend class KuduPartitionerBuilder;
explicit KuduPartitioner(Data* data);
http://git-wip-us.apache.org/repos/asf/kudu/blob/f2089195/src/kudu/tools/kudu-tool-test.cc
--
diff --git a/src/kudu/tools/kudu-tool-test.cc b/src/kudu/tools/kudu-tool-test.cc
index 12bd979..57bc295 100644
--- a/src/kudu/tools/kudu-tool-test.cc
+++ b/src/kudu/tools/kudu-tool-test.cc
@@ -2075,6 +2075,11 @@ TEST_F(ToolTest, TestHmsUpgrade) {
ASSERT_EQ(2, table_names.size());
}
+ // Restart external mini cluster to enable Hive Metastore integration.
+ cluster_->EnableMetastoreIntegration();
+ cluster_->ShutdownNodes(cluster::ClusterNodes::ALL);
+ ASSERT_OK(cluster_->Restart());
+
// Upgrade the historical metadata in both Hive Metastore and Kudu.
string out;
NO_FATALS(RunActionStdinStdoutString(
http://git-wip-us.apache.org/repos/asf/kudu/blob/f2089195/src/kudu/tools/tool_action_hms.cc
--
diff --git a/src/kudu/tools/tool_action_hms.cc
b/src/kudu/tools/tool_action_hms.cc
index 1f141c0..5c90c44 100644
--- a/src/kudu/tools/tool_action_hms.cc
+++ b/src/kudu/tools/tool_action_hms.cc
@@ -102,6 +102,16 @@ string RenameHiveIncompatibleTable(const string& name) {
return table_name;
}
+// Only alter the table in Kudu but not in the Hive Metastore.
+Status AlterKuduTable(const client::sp::shared_ptr& kudu_client,
+ const string& name,
+ const string& new_name) {
+ unique_ptr alterer(kudu_client->NewTableAlterer(name));
+ return alterer->RenameTo(new_name)
+->alter_external_catalogs(false)
+->Alter();
+}
+
// Alter legacy tables (which includes non-Impala tables, Impala
managed/external
// tables) to follow the format 'database_name.table_name' in table naming in
Kudu.
// Also, create HMS entries for non-Impala tables.
@@ -126,13 +136,6 @@ Status AlterLegacyKuduTables(const
client::sp::shared_ptr& kudu_clie
}
}
- auto alter_kudu_table = [&](const string& name,
- const string& new_name) -> Status {
-unique_ptr
[1/2] kudu git commit: KUDU-2191 (12/n): Hive Metastore notification log event listener
Repository: kudu
Updated Branches:
refs/heads/master b5f3d1a10 -> 5f1ca32f3
http://git-wip-us.apache.org/repos/asf/kudu/blob/5f1ca32f/src/kudu/master/hms_notification_log_listener.h
--
diff --git a/src/kudu/master/hms_notification_log_listener.h
b/src/kudu/master/hms_notification_log_listener.h
new file mode 100644
index 000..c4e152d
--- /dev/null
+++ b/src/kudu/master/hms_notification_log_listener.h
@@ -0,0 +1,157 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include
+#include
+
+#include "kudu/gutil/port.h"
+#include "kudu/gutil/ref_counted.h"
+#include "kudu/util/condition_variable.h"
+#include "kudu/util/mutex.h"
+#include "kudu/util/status.h"
+#include "kudu/util/status_callback.h"
+
+namespace hive {
+class NotificationEvent;
+}
+
+namespace kudu {
+
+class MonoTime;
+class Thread;
+
+namespace master {
+
+class CatalogManager;
+
+// A CatalogManager background task which listens for events occurring in the
+// Hive Metastore, and synchronizes the Kudu catalog accordingly.
+//
+// As a background task, the lifetime of an instance of this class must be less
+// than the catalog manager it belongs to.
+//
+// The notification log listener task continuously wakes up according to its
+// configured poll period, however it performs no work when the master is a
+// follower.
+//
+// When a change to the Kudu catalog is performed in response to a notification
+// log event, the corresponding event ID is recorded in the sys catalog as the
+// latest handled event. This ensures that masters do not double-apply
+// notification events as leadership changes.
+//
+// The notification log listener listens for two types of events on Kudu
tables:
+//
+// - ALTER TABLE RENAME
+//Table rename is a special case of ALTER TABLE. The notification log
+//listener listens for rename event notifications for Kudu tables, and
+//renames the corresponding Kudu table. See below for why renames can be
+//applied back to Kudu, but not other types of alterations.
+//
+// - DROP TABLE
+//The notification log listener listens for drop table events for Kudu
+//tables, and drops the corresponding Kudu table. This allows the catalogs
+//to stay synchronized when DROP TABLE and DROP DATABASE CASCADE Hive
+//commands are executed.
+//
+// The notification log listener can support renaming and dropping tables in a
+// safe manner because the Kudu table ID is stored in the HMS table entry.
Using
+// the Kudu table ID, the exact table which the event applies to can always be
+// identified. For other changes made in ALTER TABLE statements, such as ALTER
+// TABLE DROP COLUMN, there is no way to identify with certainty which column
+// has been dropped, since we do not store column IDs in the HMS table entries.
+class HmsNotificationLogListenerTask {
+ public:
+
+ explicit HmsNotificationLogListenerTask(CatalogManager* catalog_manager);
+ ~HmsNotificationLogListenerTask();
+
+ // Initializes the HMS notification log listener. When invoking this method,
+ // the catalog manager must be in the process of initializing.
+ Status Init() WARN_UNUSED_RESULT;
+
+ // Shuts down the HMS notification log listener. This must be called before
+ // shutting down the catalog manager.
+ void Shutdown();
+
+ // Waits for the notification log listener to process the latest notification
+ // log event.
+ //
+ // Note: an error will be returned if the listener is unable to retrieve the
+ // latest notifications from the HMS. If individual notifications are unable
+ // to be processed, no error will be returned.
+ Status WaitForCatchUp(const MonoTime& deadline) WARN_UNUSED_RESULT;
+
+ private:
+
+ // Runs the main loop of the listening thread.
+ void RunLoop();
+
+ // Polls the Hive Metastore for notification events, and handle them.
+ Status Poll();
+
+ // Handles an ALTER TABLE event. Must only be called on the listening thread.
+ //
+ // The event is parsed, and if it is a rename table event for a Kudu table,
+ // the table is renamed in the local catalog. All other events are ignored.
+ Status HandleAlterTableEvent(const
