hive git commit: HIVE-20776 : Run HMS filterHooks on server-side in addition to client-side (Na Li reviewed by Karthik, Sergio, Morio, Adam and Vihang Karajgaonkar)
Repository: hive
Updated Branches:
refs/heads/master d1460174d -> dfd63d979
HIVE-20776 : Run HMS filterHooks on server-side in addition to client-side (Na
Li reviewed by Karthik, Sergio, Morio, Adam and Vihang Karajgaonkar)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/dfd63d97
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/dfd63d97
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/dfd63d97
Branch: refs/heads/master
Commit: dfd63d97902b359e1643e955a4d070ac983debd5
Parents: d146017
Author: Na Li
Authored: Tue Jan 22 10:43:25 2019 -0800
Committer: Vihang Karajgaonkar
Committed: Tue Jan 22 11:08:02 2019 -0800
--
.../hive/metastore/HiveMetaStoreClient.java | 141 ---
.../hive/metastore/conf/MetastoreConf.java | 8 +-
.../hive/metastore/utils/FilterUtils.java | 375 +++
.../hadoop/hive/metastore/HiveMetaStore.java| 168 -
.../hadoop/hive/metastore/TestFilterHooks.java | 304 ++-
.../metastore/client/TestListPartitions.java| 4 +-
6 files changed, 849 insertions(+), 151 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hive/blob/dfd63d97/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
--
diff --git
a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
index 19bd9ba..30edc56 100644
---
a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
+++
b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
@@ -19,7 +19,9 @@
package org.apache.hadoop.hive.metastore;
import static org.apache.hadoop.hive.metastore.Warehouse.DEFAULT_DATABASE_NAME;
+import static org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.CAT_NAME;
import static
org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.getDefaultCatalog;
+import static
org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.parseDbName;
import static
org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.prependCatalogToDbName;
import java.io.IOException;
@@ -65,6 +67,7 @@ import org.apache.hadoop.hive.metastore.hooks.URIResolverHook;
import org.apache.hadoop.hive.metastore.partition.spec.PartitionSpecProxy;
import org.apache.hadoop.hive.metastore.security.HadoopThriftAuthBridge;
import org.apache.hadoop.hive.metastore.txn.TxnCommonUtils;
+import org.apache.hadoop.hive.metastore.utils.FilterUtils;
import org.apache.hadoop.hive.metastore.utils.JavaUtils;
import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils;
import org.apache.hadoop.hive.metastore.utils.ObjectPair;
@@ -127,6 +130,7 @@ public class HiveMetaStoreClient implements
IMetaStoreClient, AutoCloseable {
private String tokenStrForm;
private final boolean localMetaStore;
private final MetaStoreFilterHook filterHook;
+ private final boolean isClientFilterEnabled;
private final URIResolverHook uriResolverHook;
private final int fileMetadataBatchSize;
@@ -164,6 +168,7 @@ public class HiveMetaStoreClient implements
IMetaStoreClient, AutoCloseable {
}
version = MetastoreConf.getBoolVar(conf, ConfVars.HIVE_IN_TEST) ?
TEST_VERSION : VERSION;
filterHook = loadFilterHooks();
+isClientFilterEnabled = getIfClientFilterEnabled();
uriResolverHook = loadUriResolverHook();
fileMetadataBatchSize = MetastoreConf.getIntVar(
conf, ConfVars.BATCH_RETRIEVE_OBJECTS_MAX);
@@ -276,6 +281,12 @@ public class HiveMetaStoreClient implements
IMetaStoreClient, AutoCloseable {
return null;
}
+ private boolean getIfClientFilterEnabled() {
+boolean isEnabled = MetastoreConf.getBoolVar(conf,
ConfVars.METASTORE_CLIENT_FILTER_ENABLED);
+LOG.info("HMS client filtering is " + (isEnabled?"enabled.":"disabled."));
+
+return isEnabled;
+ }
private void resolveUris() throws MetaException {
String thriftUris = MetastoreConf.getVar(conf, ConfVars.THRIFT_URIS);
String serviceDiscoveryMode = MetastoreConf.getVar(conf,
ConfVars.THRIFT_SERVICE_DISCOVERY_MODE);
@@ -731,13 +742,15 @@ public class HiveMetaStoreClient implements
IMetaStoreClient, AutoCloseable {
@Override
public Catalog getCatalog(String catName) throws TException {
GetCatalogResponse rsp = client.get_catalog(new
GetCatalogRequest(catName));
-return rsp == null ? null : filterHook.filterCatalog(rsp.getCatalog());
+return rsp == null ?
+null : FilterUtils.filterCatalogIfEnabled(isClientFilterEnabled,
filterHook, rsp.getCatalog());
}
hive git commit: HIVE-21077 : Database and Catalogs should have creation time (Vihang Karajgaonkar reviewed by Karthik Manamcheri and Bharath Krishna)
Repository: hive
Updated Branches:
refs/heads/branch-3 4c73511f3 -> 1685dc311
HIVE-21077 : Database and Catalogs should have creation time (Vihang
Karajgaonkar reviewed by Karthik Manamcheri and Bharath Krishna)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/1685dc31
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/1685dc31
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/1685dc31
Branch: refs/heads/branch-3
Commit: 1685dc3113fdb731d85e6c06a873b1ad8f1ff36d
Parents: 4c73511
Author: Vihang Karajgaonkar
Authored: Thu Jan 3 10:56:05 2019 -0800
Committer: Vihang Karajgaonkar
Committed: Tue Jan 22 10:29:34 2019 -0800
--
.../gen/thrift/gen-cpp/hive_metastore_types.cpp | 44
.../gen/thrift/gen-cpp/hive_metastore_types.h | 24 +++-
.../hadoop/hive/metastore/api/Catalog.java | 111 ++-
.../hadoop/hive/metastore/api/Database.java | 111 ++-
.../src/gen/thrift/gen-php/metastore/Types.php | 46
.../gen/thrift/gen-py/hive_metastore/ttypes.py | 30 -
.../gen/thrift/gen-rb/hive_metastore_types.rb | 8 +-
.../hadoop/hive/metastore/HiveMetaStore.java| 18 ++-
.../hive/metastore/MetaStoreDirectSql.java | 3 +-
.../hadoop/hive/metastore/ObjectStore.java | 4 +
.../client/builder/DatabaseBuilder.java | 7 ++
.../hadoop/hive/metastore/model/MCatalog.java | 9 ++
.../hadoop/hive/metastore/model/MDatabase.java | 9 ++
.../src/main/resources/package.jdo | 8 ++
.../main/sql/derby/hive-schema-3.2.0.derby.sql | 6 +-
.../sql/derby/upgrade-3.1.0-to-3.2.0.derby.sql | 4 +
.../main/sql/mssql/hive-schema-3.2.0.mssql.sql | 6 +-
.../sql/mssql/upgrade-3.1.0-to-3.2.0.mssql.sql | 4 +
.../main/sql/mysql/hive-schema-3.2.0.mysql.sql | 2 +
.../sql/mysql/upgrade-3.1.0-to-3.2.0.mysql.sql | 4 +
.../sql/oracle/hive-schema-3.2.0.oracle.sql | 3 +-
.../oracle/upgrade-3.1.0-to-3.2.0.oracle.sql| 4 +
.../sql/postgres/hive-schema-3.2.0.postgres.sql | 6 +-
.../upgrade-3.1.0-to-3.2.0.postgres.sql | 4 +
.../src/main/thrift/hive_metastore.thrift | 10 +-
.../hive/metastore/cache/TestCachedStore.java | 1 +
.../hive/metastore/client/TestCatalogs.java | 2 +
.../hive/metastore/client/TestDatabases.java| 29 +
.../tools/TestSchemaToolForMetastore.java | 27 +++--
29 files changed, 499 insertions(+), 45 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hive/blob/1685dc31/standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.cpp
--
diff --git
a/standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.cpp
b/standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.cpp
index 26420dd..b848718 100644
--- a/standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.cpp
+++ b/standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.cpp
@@ -4456,6 +4456,11 @@ void Catalog::__set_locationUri(const std::string& val) {
this->locationUri = val;
}
+void Catalog::__set_createTime(const int32_t val) {
+ this->createTime = val;
+__isset.createTime = true;
+}
+
uint32_t Catalog::read(::apache::thrift::protocol::TProtocol* iprot) {
apache::thrift::protocol::TInputRecursionTracker tracker(*iprot);
@@ -4501,6 +4506,14 @@ uint32_t
Catalog::read(::apache::thrift::protocol::TProtocol* iprot) {
xfer += iprot->skip(ftype);
}
break;
+ case 4:
+if (ftype == ::apache::thrift::protocol::T_I32) {
+ xfer += iprot->readI32(this->createTime);
+ this->__isset.createTime = true;
+} else {
+ xfer += iprot->skip(ftype);
+}
+break;
default:
xfer += iprot->skip(ftype);
break;
@@ -4531,6 +4544,11 @@ uint32_t
Catalog::write(::apache::thrift::protocol::TProtocol* oprot) const {
xfer += oprot->writeString(this->locationUri);
xfer += oprot->writeFieldEnd();
+ if (this->__isset.createTime) {
+xfer += oprot->writeFieldBegin("createTime",
::apache::thrift::protocol::T_I32, 4);
+xfer += oprot->writeI32(this->createTime);
+xfer += oprot->writeFieldEnd();
+ }
xfer += oprot->writeFieldStop();
xfer += oprot->writeStructEnd();
return xfer;
@@ -4541,6 +4559,7 @@ void swap(Catalog &a, Catalog &b) {
swap(a.name, b.name);
swap(a.description, b.description);
swap(a.locationUri, b.locationUri);
+ swap(a.createTime, b.createTime);
swap(a.__isset, b.__isset);
}
@@ -4548,12 +4567,14 @@ Catalog::Catalog(const Catalog& other130) {
name = other130.name;
description = other130.description;
locationUri = other130.locationUri;
+ createTime = other130.createTime;
__isset = other130.__isset;
}
C
hive git commit: HIVE-21077 : Database and Catalogs should have creation time (Vihang Karajgaonkar reviewed by Karthik Manamcheri and Bharath Krishna)
Repository: hive
Updated Branches:
refs/heads/master cb74a685c -> d1460174d
HIVE-21077 : Database and Catalogs should have creation time (Vihang
Karajgaonkar reviewed by Karthik Manamcheri and Bharath Krishna)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/d1460174
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/d1460174
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/d1460174
Branch: refs/heads/master
Commit: d1460174d5322bcfb642b489dd250c59770f0551
Parents: cb74a68
Author: Vihang Karajgaonkar
Authored: Thu Jan 3 10:56:05 2019 -0800
Committer: Vihang Karajgaonkar
Committed: Tue Jan 22 10:17:53 2019 -0800
--
.../hadoop/hive/metastore/api/Catalog.java | 111 ++-
.../hadoop/hive/metastore/api/Database.java | 111 ++-
.../src/gen/thrift/gen-php/metastore/Types.php | 46
.../gen/thrift/gen-py/hive_metastore/ttypes.py | 30 -
.../gen/thrift/gen-rb/hive_metastore_types.rb | 8 +-
.../src/main/thrift/hive_metastore.thrift | 10 +-
.../hadoop/hive/metastore/HiveMetaStore.java| 18 ++-
.../hive/metastore/MetaStoreDirectSql.java | 3 +-
.../hadoop/hive/metastore/ObjectStore.java | 4 +
.../client/builder/DatabaseBuilder.java | 7 ++
.../hadoop/hive/metastore/model/MCatalog.java | 9 ++
.../hadoop/hive/metastore/model/MDatabase.java | 9 ++
.../src/main/resources/package.jdo | 8 ++
.../main/sql/derby/hive-schema-3.2.0.derby.sql | 6 +-
.../main/sql/derby/hive-schema-4.0.0.derby.sql | 6 +-
.../sql/derby/upgrade-3.1.0-to-3.2.0.derby.sql | 4 +
.../sql/derby/upgrade-3.2.0-to-4.0.0.derby.sql | 1 -
.../main/sql/mssql/hive-schema-3.2.0.mssql.sql | 6 +-
.../main/sql/mssql/hive-schema-4.0.0.mssql.sql | 6 +-
.../sql/mssql/upgrade-3.1.0-to-3.2.0.mssql.sql | 4 +
.../main/sql/mysql/hive-schema-3.2.0.mysql.sql | 2 +
.../main/sql/mysql/hive-schema-4.0.0.mysql.sql | 2 +
.../sql/mysql/upgrade-3.1.0-to-3.2.0.mysql.sql | 4 +
.../sql/oracle/hive-schema-3.2.0.oracle.sql | 4 +-
.../sql/oracle/hive-schema-4.0.0.oracle.sql | 4 +-
.../oracle/upgrade-3.1.0-to-3.2.0.oracle.sql| 3 +
.../sql/postgres/hive-schema-3.2.0.postgres.sql | 6 +-
.../sql/postgres/hive-schema-4.0.0.postgres.sql | 6 +-
.../upgrade-3.1.0-to-3.2.0.postgres.sql | 4 +
.../hive/metastore/cache/TestCachedStore.java | 1 +
.../hive/metastore/client/TestCatalogs.java | 2 +
.../hive/metastore/client/TestDatabases.java| 29 +
.../tools/TestSchemaToolForMetastore.java | 27 +++--
33 files changed, 452 insertions(+), 49 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hive/blob/d1460174/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/Catalog.java
--
diff --git
a/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/Catalog.java
b/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/Catalog.java
index 3eb4dbd..3e968dc 100644
---
a/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/Catalog.java
+++
b/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/Catalog.java
@@ -41,6 +41,7 @@ import org.slf4j.LoggerFactory;
private static final org.apache.thrift.protocol.TField NAME_FIELD_DESC = new
org.apache.thrift.protocol.TField("name",
org.apache.thrift.protocol.TType.STRING, (short)1);
private static final org.apache.thrift.protocol.TField
DESCRIPTION_FIELD_DESC = new org.apache.thrift.protocol.TField("description",
org.apache.thrift.protocol.TType.STRING, (short)2);
private static final org.apache.thrift.protocol.TField
LOCATION_URI_FIELD_DESC = new org.apache.thrift.protocol.TField("locationUri",
org.apache.thrift.protocol.TType.STRING, (short)3);
+ private static final org.apache.thrift.protocol.TField
CREATE_TIME_FIELD_DESC = new org.apache.thrift.protocol.TField("createTime",
org.apache.thrift.protocol.TType.I32, (short)4);
private static final Map, SchemeFactory> schemes =
new HashMap, SchemeFactory>();
static {
@@ -51,12 +52,14 @@ import org.slf4j.LoggerFactory;
private String name; // required
private String description; // optional
private String locationUri; // required
+ private int createTime; // optional
/** The set of fields this struct contains, along with convenience methods
for finding and manipulating them. */
public enum _Fields implements org.apache.thrift.TFieldIdEnum {
NAME((short)1, "name"),
DESCRIPTION((short)2, "description"),
-
