This is an automated email from the ASF dual-hosted git repository.
abukor pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kudu.git
The following commit(s) were added to refs/heads/master by this push:
new f392503 KUDU-2973 Add semi-database support for Ranger
f392503 is described below
commit f392503972e41fe57d80fefb3779a82c9a92229f
Author: Attila Bukor <[email protected]>
AuthorDate: Thu Feb 6 17:27:34 2020 +0100
KUDU-2973 Add semi-database support for Ranger
The table names are "normalized" if HMS integration is enabled so that
they can be synchronized to HMS. Hive's table/database identifiers are
pretty restrictive (alphanumerical ASCII characters + underscore and
forward slash) and it's case-insensitive. As Kudu doesn't support
databases it imposes a further restriction: the table identifiers to be
synchronized with HMS have to contain exactly one "." character which
separates the database name from the table name in HMS.
Ranger's restrictions regarding identifiers are more lax, can be case
sensitive (will not be used for Kudu) and is not limited to ASCII.
This commit introduces a method to parse database and table name for
Ranger from a table identifier that works slightly different from Hive's
table name parser, as it allows non-ASCII characters and provides a
configurable default database name when the table name doesn't contain a
period.
Change-Id: I11431ff5bc75540edff56ef3d4ad384fa37d33d5
Reviewed-on: http://gerrit.cloudera.org:8080/15018
Tested-by: Kudu Jenkins
Reviewed-by: Alexey Serbin <[email protected]>
Reviewed-by: Hao Hao <[email protected]>
---
src/kudu/common/table_util-test.cc | 82 +++++++++++++++++++++++++-
src/kudu/common/table_util.cc | 57 ++++++++++++++++--
src/kudu/common/table_util.h | 17 +++++-
src/kudu/integration-tests/master_hms-itest.cc | 4 +-
4 files changed, 152 insertions(+), 8 deletions(-)
diff --git a/src/kudu/common/table_util-test.cc
b/src/kudu/common/table_util-test.cc
index b351ebe..f443205 100644
--- a/src/kudu/common/table_util-test.cc
+++ b/src/kudu/common/table_util-test.cc
@@ -51,7 +51,7 @@ TEST(TestTableUtil, TestParseHiveTableIdentifier) {
EXPECT_TRUE(ParseHiveTableIdentifier("", &db, &tbl).IsInvalidArgument());
EXPECT_TRUE(ParseHiveTableIdentifier(".", &db, &tbl).IsInvalidArgument());
- EXPECT_TRUE(ParseHiveTableIdentifier("no-table", &db,
&tbl).IsInvalidArgument());
+ EXPECT_TRUE(ParseHiveTableIdentifier("no_table", &db,
&tbl).IsInvalidArgument());
EXPECT_TRUE(ParseHiveTableIdentifier("lots.of.tables", &db,
&tbl).IsInvalidArgument());
EXPECT_TRUE(ParseHiveTableIdentifier(".no_table", &db,
&tbl).IsInvalidArgument());
EXPECT_TRUE(ParseHiveTableIdentifier("no_table.", &db,
&tbl).IsInvalidArgument());
@@ -62,4 +62,84 @@ TEST(TestTableUtil, TestParseHiveTableIdentifier) {
EXPECT_TRUE(ParseHiveTableIdentifier(string("\0.\0", 3), &db,
&tbl).IsInvalidArgument());
}
+TEST(TestTableUtil, TestRangerTableIdentifier) {
+ string db;
+ Slice tbl;
+ string table;
+ bool default_database;
+
+ table = "foo.bar";
+ EXPECT_OK(ParseRangerTableIdentifier(table, &db, &tbl, &default_database));
+ EXPECT_EQ("foo", db);
+ EXPECT_EQ("bar", tbl);
+ EXPECT_FALSE(default_database);
+
+ table = "99bottles.my_awesome/table/22";
+ EXPECT_OK(ParseRangerTableIdentifier(table, &db, &tbl, &default_database));
+ EXPECT_EQ("99bottles", db);
+ EXPECT_EQ("my_awesome/table/22", tbl);
+ EXPECT_FALSE(default_database);
+
+ table = "99/bottles.my_awesome/table/22";
+ EXPECT_OK(ParseRangerTableIdentifier(table, &db, &tbl, &default_database));
+ EXPECT_EQ("99/bottles", db);
+ EXPECT_EQ("my_awesome/table/22", tbl);
+ EXPECT_FALSE(default_database);
+
+ table = "_leading_underscore.trailing_underscore_";
+ EXPECT_OK(ParseRangerTableIdentifier(table, &db, &tbl, &default_database));
+ EXPECT_EQ("_leading_underscore", db);
+ EXPECT_EQ("trailing_underscore_", tbl);
+ EXPECT_FALSE(default_database);
+
+ table = "foo";
+ EXPECT_OK(ParseRangerTableIdentifier(table, &db, &tbl, &default_database));
+ EXPECT_EQ("default", db);
+ EXPECT_EQ("foo", tbl);
+ EXPECT_TRUE(default_database);
+
+ table = "default.foo";
+ EXPECT_OK(ParseRangerTableIdentifier(table, &db, &tbl, &default_database));
+ EXPECT_EQ("default", db);
+ EXPECT_EQ("foo", tbl);
+ EXPECT_FALSE(default_database);
+
+ table = "lots.of.tables";
+ EXPECT_OK(ParseRangerTableIdentifier(table, &db, &tbl, &default_database));
+ EXPECT_EQ("lots", db);
+ EXPECT_EQ("of.tables", tbl);
+ EXPECT_FALSE(default_database);
+
+ table = "db_name..table_name";
+ EXPECT_OK(ParseRangerTableIdentifier(table, &db, &tbl, &default_database));
+ EXPECT_EQ("db_name", db);
+ EXPECT_EQ(".table_name", tbl);
+ EXPECT_FALSE(default_database);
+
+ EXPECT_TRUE(ParseRangerTableIdentifier("", &db, &tbl, &default_database)
+ .IsInvalidArgument());
+ EXPECT_TRUE(ParseRangerTableIdentifier(".", &db, &tbl, &default_database)
+ .IsInvalidArgument());
+ EXPECT_OK(ParseRangerTableIdentifier("no_table", &db, &tbl,
+ &default_database));
+ EXPECT_OK(ParseRangerTableIdentifier("lots.of.tables", &db, &tbl,
+ &default_database));
+ EXPECT_TRUE(ParseRangerTableIdentifier("no_table.", &db, &tbl,
+ &default_database)
+ .IsInvalidArgument());
+ EXPECT_TRUE(ParseRangerTableIdentifier(".no_database", &db, &tbl,
+ &default_database)
+ .IsInvalidArgument());
+ EXPECT_OK(ParseRangerTableIdentifier("punctuation?.yes", &db, &tbl,
+ &default_database));
+ EXPECT_OK(ParseRangerTableIdentifier("white space.yes", &db, &tbl,
+ &default_database));
+ EXPECT_OK(ParseRangerTableIdentifier("unicode☃tables.yes", &db, &tbl,
+ &default_database));
+ EXPECT_OK(ParseRangerTableIdentifier("unicode.☃tables.yes", &db, &tbl,
+ &default_database));
+ EXPECT_OK(ParseRangerTableIdentifier(string("\0.\0", 3), &db, &tbl,
+ &default_database));
+}
+
} // namespace kudu
diff --git a/src/kudu/common/table_util.cc b/src/kudu/common/table_util.cc
index bc65fc8..56db851 100644
--- a/src/kudu/common/table_util.cc
+++ b/src/kudu/common/table_util.cc
@@ -20,24 +20,38 @@
#include <string>
#include <boost/optional/optional.hpp>
+#include <gflags/gflags.h>
#include "kudu/gutil/strings/charset.h"
#include "kudu/util/slice.h"
#include "kudu/util/status.h"
-using boost::optional;
+DEFINE_string(ranger_default_database, "default",
+ "Name of the default database which is used in the Ranger "
+ "authorization context when the database name is not specified "
+ "in the table name. Ranger makes no difference between "
+ "<ranger_default_database>.<table> and <table>, so privileges "
+ "granted on <table> in <ranger_default_database> are applied to"
+ "both <ranger_default_database>.<table> and <table> in Kudu.");
+
using std::string;
namespace kudu {
-const char* const kInvalidTableError = "when the Hive Metastore integration "
+const char* const kInvalidHiveTableError = "when the Hive Metastore
integration "
"is enabled, Kudu table names must be a period ('.') separated database
and table name "
"identifier pair, each containing only ASCII alphanumeric characters, '_',
and '/'";
+const char* const kInvalidRangerTableError = "when Ranger authorization is
enabled, "
+ "Kudu table names must not begin with a period ('.') and if they contain a
period, there "
+ "must be other characters after the first one, as the first period is
treated as a separator "
+ "between the database and table name. The table and the database name
can't be empty.";
+
+const char kSeparator = '.';
+
Status ParseHiveTableIdentifier(const string& table_name,
Slice* hms_database,
Slice* hms_table) {
- const char kSeparator = '.';
strings::CharSet charset("abcdefghijklmnopqrstuvwxyz"
"ABCDEFGHIJKLMNOPQRSTUVWXYZ"
"0123456789"
@@ -50,12 +64,12 @@ Status ParseHiveTableIdentifier(const string& table_name,
if (c == kSeparator && !separator_idx) {
separator_idx = idx;
} else {
- return Status::InvalidArgument(kInvalidTableError, table_name);
+ return Status::InvalidArgument(kInvalidHiveTableError, table_name);
}
}
}
if (!separator_idx || *separator_idx == 0 || *separator_idx ==
table_name.size() - 1) {
- return Status::InvalidArgument(kInvalidTableError, table_name);
+ return Status::InvalidArgument(kInvalidHiveTableError, table_name);
}
*hms_database = Slice(table_name.data(), *separator_idx);
@@ -64,4 +78,37 @@ Status ParseHiveTableIdentifier(const string& table_name,
return Status::OK();
}
+Status ParseRangerTableIdentifier(const string& table_name,
+ string* ranger_database,
+ Slice* ranger_table,
+ bool* default_database) {
+ auto separator_idx = boost::make_optional<int>(false, 0);
+ for (int idx = 0; idx < table_name.size(); ++idx) {
+ char c = table_name[idx];
+ if (c == kSeparator) {
+ separator_idx = idx;
+ break;
+ }
+ }
+
+ if (separator_idx) {
+ if (*separator_idx == 0 || *separator_idx == table_name.size() - 1) {
+ return Status::InvalidArgument(kInvalidRangerTableError, table_name);
+ }
+ *ranger_database = table_name.substr(0, *separator_idx);
+ *ranger_table = Slice(table_name.data() + *separator_idx + 1,
+ table_name.size() - *separator_idx - 1);
+ } else {
+ *ranger_database = FLAGS_ranger_default_database;
+ *ranger_table = Slice(table_name.data());
+ }
+ *default_database = !separator_idx;
+
+ if (ranger_table->empty()) {
+ return Status::InvalidArgument(kInvalidRangerTableError, table_name);
+ }
+
+ return Status::OK();
+}
+
} // namespace kudu
diff --git a/src/kudu/common/table_util.h b/src/kudu/common/table_util.h
index d68076b..a9d2e8a 100644
--- a/src/kudu/common/table_util.h
+++ b/src/kudu/common/table_util.h
@@ -25,7 +25,9 @@ namespace kudu {
class Slice;
-extern const char* const kInvalidTableError;
+extern const char* const kInvalidHiveTableError;
+
+extern const char* const kInvalidRangerTableError;
// Parses a Kudu table name of the form '<database>.<table>' into
// a Hive database and table name. Returns an error if the Kudu
@@ -35,4 +37,17 @@ Status ParseHiveTableIdentifier(const std::string&
table_name,
Slice* hms_database,
Slice* hms_table) WARN_UNUSED_RESULT;
+// Parses a Kudu table name of the form '<database>.<table>' into a
+// Ranger database and table name. If the table name doesn't contain a period
it
+// defaults to a configurable default database name. If there are multiple
+// periods in the table name the first one will separate the database name from
+// the table name. The returned 'default_database' bool indicates if the
default
+// database name was used (if a database name is provided in the table name but
+// it is the same as the default database it will be false). The returned
+// 'ranger_table' slice must not outlive 'table_name'.
+Status ParseRangerTableIdentifier(const std::string& table_name,
+ std::string* ranger_database,
+ Slice* ranger_table,
+ bool* default_database) WARN_UNUSED_RESULT;
+
} // namespace kudu
diff --git a/src/kudu/integration-tests/master_hms-itest.cc
b/src/kudu/integration-tests/master_hms-itest.cc
index 9680836..3b3f026 100644
--- a/src/kudu/integration-tests/master_hms-itest.cc
+++ b/src/kudu/integration-tests/master_hms-itest.cc
@@ -16,9 +16,11 @@
// under the License.
#include <algorithm>
+#include <initializer_list>
#include <map>
#include <memory>
#include <string>
+#include <utility>
#include <vector>
#include <boost/optional/optional.hpp>
@@ -173,7 +175,7 @@ TEST_F(MasterHmsTest, TestRenameTable) {
table_alterer.reset(client_->NewTableAlterer("db.a"));
s = table_alterer->RenameTo("foo")->Alter();
ASSERT_TRUE(s.IsInvalidArgument()) << s.ToString();
- ASSERT_STR_CONTAINS(s.ToString(), kInvalidTableError);
+ ASSERT_STR_CONTAINS(s.ToString(), kInvalidHiveTableError);
// Attempt to rename the Kudu table to a non-existent database.
table_alterer.reset(client_->NewTableAlterer("db.a"));
