[hive] branch master updated: HIVE-14737: Problem accessing /logs in a Kerberized Hive Server 2 Web UI (Rajkumar Singh, reviewed by Daniel Dai)
This is an automated email from the ASF dual-hosted git repository. daijy pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/hive.git The following commit(s) were added to refs/heads/master by this push: new 22a5e21 HIVE-14737: Problem accessing /logs in a Kerberized Hive Server 2 Web UI (Rajkumar Singh, reviewed by Daniel Dai) 22a5e21 is described below commit 22a5e211df38a5de1cc3d1e6b15c52389aace2e1 Author: Daniel Dai AuthorDate: Wed Jun 26 21:05:38 2019 -0700 HIVE-14737: Problem accessing /logs in a Kerberized Hive Server 2 Web UI (Rajkumar Singh, reviewed by Daniel Dai) --- common/src/java/org/apache/hive/http/HttpServer.java | 10 ++ 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/common/src/java/org/apache/hive/http/HttpServer.java b/common/src/java/org/apache/hive/http/HttpServer.java index bbcc67e..8968529 100644 --- a/common/src/java/org/apache/hive/http/HttpServer.java +++ b/common/src/java/org/apache/hive/http/HttpServer.java @@ -462,7 +462,7 @@ public class HttpServer { /** * Secure the web server with kerberos (AuthenticationFilter). */ - void setupSpnegoFilter(Builder b) throws IOException { + void setupSpnegoFilter(Builder b, ServletContextHandler ctx) throws IOException { Map params = new HashMap(); params.put("kerberos.principal", SecurityUtil.getServerPrincipal(b.spnegoPrincipal, b.host)); @@ -471,8 +471,7 @@ public class HttpServer { FilterHolder holder = new FilterHolder(); holder.setClassName(AuthenticationFilter.class.getName()); holder.setInitParameters(params); - -ServletHandler handler = webAppContext.getServletHandler(); +ServletHandler handler = ctx.getServletHandler(); handler.addFilterWithMapping( holder, "/*", FilterMapping.ALL); } @@ -565,7 +564,7 @@ public class HttpServer { if (b.useSPNEGO) { // Secure the web server with kerberos - setupSpnegoFilter(b); + setupSpnegoFilter(b, webAppContext); } if (b.enableCORS) { @@ -648,6 +647,9 @@ public class HttpServer { ServletContextHandler logCtx = new ServletContextHandler(contexts, "/logs"); setContextAttributes(logCtx.getServletContext(), b.contextAttrs); + if(b.useSPNEGO) { +setupSpnegoFilter(b,logCtx); + } logCtx.addServlet(AdminAuthorizedServlet.class, "/*"); logCtx.setResourceBase(logDir); logCtx.setDisplayName("logs");
[hive] branch master updated: HIVE-21752 Thread Safety and Memory Leaks in HCatRecordObjectInspectorFactory (Jalpan Randeri via Alan Gates)
This is an automated email from the ASF dual-hosted git repository. gates pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/hive.git The following commit(s) were added to refs/heads/master by this push: new ee3aeb7 HIVE-21752 Thread Safety and Memory Leaks in HCatRecordObjectInspectorFactory (Jalpan Randeri via Alan Gates) ee3aeb7 is described below commit ee3aeb77dbde7f106dd89edc8768e0d40830de0d Author: Alan Gates AuthorDate: Wed Jun 26 08:40:20 2019 -0700 HIVE-21752 Thread Safety and Memory Leaks in HCatRecordObjectInspectorFactory (Jalpan Randeri via Alan Gates) --- .../data/HCatRecordObjectInspectorFactory.java | 27 +++--- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/data/HCatRecordObjectInspectorFactory.java b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/data/HCatRecordObjectInspectorFactory.java index 18bf3a4..c351681 100644 --- a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/data/HCatRecordObjectInspectorFactory.java +++ b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/data/HCatRecordObjectInspectorFactory.java @@ -19,9 +19,11 @@ package org.apache.hive.hcatalog.data; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; +import java.util.concurrent.TimeUnit; +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; import org.apache.hadoop.hive.serde2.SerDeException; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; @@ -40,11 +42,20 @@ import org.slf4j.LoggerFactory; public class HCatRecordObjectInspectorFactory { private final static Logger LOG = LoggerFactory.getLogger(HCatRecordObjectInspectorFactory.class); + private static final int INITIAL_CACHE_CAPACITY = 1024; + private static final int MAX_CACHE_CAPACITY = 10 * INITIAL_CACHE_CAPACITY; - static HashMap cachedHCatRecordObjectInspectors = -new HashMap(); - static HashMap cachedObjectInspectors = -new HashMap(); + private static final CacheBuilder boundedCache = + CacheBuilder.newBuilder() + .initialCapacity(INITIAL_CACHE_CAPACITY) + .maximumSize(MAX_CACHE_CAPACITY) + .concurrencyLevel(Runtime.getRuntime().availableProcessors()) + .expireAfterAccess(5, TimeUnit.MINUTES); + + private static final Cache cachedHCatRecordObjectInspectors + = boundedCache.build(); + private static final Cache cachedObjectInspectors + = boundedCache.build(); /** * Returns HCatRecordObjectInspector given a StructTypeInfo type definition for the record to look into @@ -54,7 +65,7 @@ public class HCatRecordObjectInspectorFactory { */ public static HCatRecordObjectInspector getHCatRecordObjectInspector( StructTypeInfo typeInfo) throws SerDeException { -HCatRecordObjectInspector oi = cachedHCatRecordObjectInspectors.get(typeInfo); +HCatRecordObjectInspector oi = cachedHCatRecordObjectInspectors.getIfPresent(typeInfo); if (oi == null) { LOG.debug("Got asked for OI for {} [{} ]", typeInfo.getCategory(), typeInfo.getTypeName()); @@ -86,7 +97,7 @@ public class HCatRecordObjectInspectorFactory { public static ObjectInspector getStandardObjectInspectorFromTypeInfo(TypeInfo typeInfo) { -ObjectInspector oi = cachedObjectInspectors.get(typeInfo); +ObjectInspector oi = cachedObjectInspectors.getIfPresent(typeInfo); if (oi == null) { LOG.debug("Got asked for OI for {}, [{}]", typeInfo.getCategory(), typeInfo.getTypeName()); @@ -123,7 +134,7 @@ public class HCatRecordObjectInspectorFactory { default: oi = null; } - cachedObjectInspectors.put(typeInfo, oi); + cachedObjectInspectors.asMap().put(typeInfo, oi); } return oi; }
[hive] 02/02: HIVE-21914: Move Function and Macro related DDL operations into the DDL framework (Miklos Gergely via Zoltan Haindrich)
This is an automated email from the ASF dual-hosted git repository. kgyrtkirk pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/hive.git commit a3ee8d0050a0ac652abd3b46022c28d75cdc3340 Author: Miklos Gergely AuthorDate: Wed Jun 26 17:34:03 2019 +0200 HIVE-21914: Move Function and Macro related DDL operations into the DDL framework (Miklos Gergely via Zoltan Haindrich) Signed-off-by: Zoltan Haindrich --- .../hive/llap/daemon/impl/FunctionLocalizer.java | 4 +- .../{plan => ddl/function}/CreateFunctionDesc.java | 66 ++-- .../ql/ddl/function/CreateFunctionOperation.java | 217 +++ .../hive/ql/ddl/function/CreateMacroDesc.java | 81 + .../function/CreateMacroOperation.java}| 38 +- .../{plan => ddl/function}/DropFunctionDesc.java | 51 +-- .../ql/ddl/function/DropFunctionOperation.java | 107 ++ .../ql/{plan => ddl/function}/DropMacroDesc.java | 28 +- .../function/DropMacroOperation.java} | 42 +-- .../function/ReloadFunctionsDesc.java} | 13 +- .../function/ReloadFunctionsOperation.java}| 43 +-- .../apache/hadoop/hive/ql/exec/FunctionTask.java | 399 - .../apache/hadoop/hive/ql/exec/FunctionUtils.java | 55 ++- .../org/apache/hadoop/hive/ql/exec/Registry.java | 4 +- .../apache/hadoop/hive/ql/exec/TaskFactory.java| 3 - .../org/apache/hadoop/hive/ql/metadata/Hive.java | 3 +- .../hive/ql/optimizer/QueryPlanPostProcessor.java | 2 - .../hive/ql/parse/FunctionSemanticAnalyzer.java| 18 +- .../org/apache/hadoop/hive/ql/parse/HiveParser.g | 11 +- .../hive/ql/parse/MacroSemanticAnalyzer.java | 12 +- .../hive/ql/parse/SemanticAnalyzerFactory.java | 4 +- .../repl/load/message/CreateFunctionHandler.java | 12 +- .../repl/load/message/DropFunctionHandler.java | 9 +- .../hadoop/hive/ql/plan/CreateMacroDesc.java | 73 .../apache/hadoop/hive/ql/plan/FunctionWork.java | 93 - .../hive/ql/parse/TestMacroSemanticAnalyzer.java | 3 + .../hadoop/hive/ql/plan/TestCreateMacroDesc.java | 8 +- .../hadoop/hive/ql/plan/TestDropMacroDesc.java | 3 +- ql/src/test/queries/clientpositive/create_func1.q | 15 + .../create_function_nonexistent_class.q.out| 2 +- .../create_function_nonudf_class.q.out | 2 +- .../clientnegative/create_unknown_genericudf.q.out | 2 +- .../clientnegative/create_unknown_udf_udaf.q.out | 2 +- .../test/results/clientnegative/ivyDownload.q.out | 2 +- .../udf_function_does_not_implement_udf.q.out | 2 +- .../clientnegative/udf_local_resource.q.out| 2 +- .../clientnegative/udf_nonexistent_resource.q.out | 2 +- .../test/results/clientpositive/create_func1.q.out | 102 ++ .../clientpositive/create_genericudaf.q.out| 4 + .../results/clientpositive/create_genericudf.q.out | 4 + .../test/results/clientpositive/create_udaf.q.out | 4 + ql/src/test/results/clientpositive/drop_udf.q.out | 3 + .../clientpositive/tez/explainanalyze_3.q.out | 2 + .../results/clientpositive/tez/explainuser_3.q.out | 2 + .../clientpositive/udf_compare_java_string.q.out | 4 + .../clientpositive/udf_logic_java_boolean.q.out| 4 + .../results/clientpositive/udf_testlength.q.out| 4 + .../results/clientpositive/udf_testlength2.q.out | 4 + 48 files changed, 778 insertions(+), 792 deletions(-) diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/FunctionLocalizer.java b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/FunctionLocalizer.java index 136fe2a..315c5be 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/FunctionLocalizer.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/FunctionLocalizer.java @@ -37,7 +37,7 @@ import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.ResourceUri; import org.apache.hadoop.hive.ql.exec.AddToClassPathAction; import org.apache.hadoop.hive.ql.exec.FunctionRegistry; -import org.apache.hadoop.hive.ql.exec.FunctionTask; +import org.apache.hadoop.hive.ql.exec.FunctionUtils; import org.apache.hadoop.hive.ql.exec.UDFClassLoader; import org.apache.hadoop.hive.ql.exec.FunctionInfo.FunctionResource; import org.apache.hadoop.hive.ql.metadata.Hive; @@ -269,7 +269,7 @@ public class FunctionLocalizer implements GenericUDFBridge.UdfWhitelistChecker { } for (ResourceUri resource : resources) { URI srcUri = ResourceDownloader.createURI(resource.getUri()); - ResourceType rt = FunctionTask.getResourceType(resource.getResourceType()); + ResourceType rt = FunctionUtils.getResourceType(resource.getResourceType()); localizeOneResource(fqfn, srcUri, rt, result); } recentlyLocalizedClasses.add(className); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/CreateFunctionDesc.java
[hive] 01/02: HIVE-21896: SHOW FUNCTIONS / SHOW FUNCTIONS LIKE - clarify (Miklos Gergely via Zoltan Haindrich)
This is an automated email from the ASF dual-hosted git repository. kgyrtkirk pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/hive.git commit bc40efdfafddee8cd39fa5526a968bc87d7eb820 Author: Miklos Gergely AuthorDate: Wed Jun 26 17:33:53 2019 +0200 HIVE-21896: SHOW FUNCTIONS / SHOW FUNCTIONS LIKE - clarify (Miklos Gergely via Zoltan Haindrich) Signed-off-by: Zoltan Haindrich --- .../TestReplicationScenariosAcrossInstances.java | 24 +- .../apache/hadoop/hive/ql/ddl/DDLOperation.java| 3 + .../hive/ql/ddl/function/ShowFunctionsDesc.java| 6 - .../ql/ddl/function/ShowFunctionsOperation.java| 7 +- .../table/storage/AlterTableSetSerdeOperation.java | 5 +- .../hadoop/hive/ql/exec/FunctionRegistry.java | 2 +- .../hadoop/hive/ql/parse/DDLSemanticAnalyzer.java | 8 +- .../org/apache/hadoop/hive/ql/parse/HiveParser.g | 2 +- .../queries/clientnegative/show_functions_wrong.q | 2 + ql/src/test/queries/clientpositive/create_func1.q | 4 +- .../clientpositive/show_describe_func_quotes.q | 4 +- .../test/queries/clientpositive/show_functions.q | 20 +- .../clientnegative/show_functions_wrong.q.out | 1 + .../test/results/clientpositive/create_func1.q.out | 8 +- .../clientpositive/show_describe_func_quotes.q.out | 8 +- .../results/clientpositive/show_functions.q.out| 347 +++-- 16 files changed, 373 insertions(+), 78 deletions(-) diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcrossInstances.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcrossInstances.java index 970b8a5..3ee8490 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcrossInstances.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcrossInstances.java @@ -91,14 +91,14 @@ public class TestReplicationScenariosAcrossInstances extends BaseReplicationAcro replica.load(replicatedDbName, incrementalDump.dumpLocation) .run("REPL STATUS " + replicatedDbName) .verifyResult(incrementalDump.lastReplicationId) -.run("SHOW FUNCTIONS LIKE '" + replicatedDbName + "*'") +.run("SHOW FUNCTIONS LIKE '" + replicatedDbName + "%'") .verifyResult(replicatedDbName + ".testFunctionOne"); // Test the idempotent behavior of CREATE FUNCTION replica.load(replicatedDbName, incrementalDump.dumpLocation) .run("REPL STATUS " + replicatedDbName) .verifyResult(incrementalDump.lastReplicationId) -.run("SHOW FUNCTIONS LIKE '" + replicatedDbName + "*'") +.run("SHOW FUNCTIONS LIKE '" + replicatedDbName + "%'") .verifyResult(replicatedDbName + ".testFunctionOne"); } @@ -148,7 +148,7 @@ public class TestReplicationScenariosAcrossInstances extends BaseReplicationAcro replica.run("use " + replicatedDbName) .run("repl status " + replicatedDbName) .verifyResult("null") -.run("show functions like '" + replicatedDbName + "*'") +.run("show functions like '" + replicatedDbName + "%'") .verifyResult(replicatedDbName + "." + funcName1); // Verify no calls to load f1 only f2. @@ -182,7 +182,7 @@ public class TestReplicationScenariosAcrossInstances extends BaseReplicationAcro replica.run("use " + replicatedDbName) .run("repl status " + replicatedDbName) .verifyResult(tuple.lastReplicationId) -.run("show functions like '" + replicatedDbName +"*'") +.run("show functions like '" + replicatedDbName +"%'") .verifyResults(new String[] {replicatedDbName + "." + funcName1, replicatedDbName +"." +funcName2}); } @@ -204,14 +204,14 @@ public class TestReplicationScenariosAcrossInstances extends BaseReplicationAcro replica.load(replicatedDbName, incrementalDump.dumpLocation) .run("REPL STATUS " + replicatedDbName) .verifyResult(incrementalDump.lastReplicationId) -.run("SHOW FUNCTIONS LIKE '*testfunctionanother*'") +.run("SHOW FUNCTIONS LIKE '%testfunctionanother%'") .verifyResult(null); // Test the idempotent behavior of DROP FUNCTION replica.load(replicatedDbName, incrementalDump.dumpLocation) .run("REPL STATUS " + replicatedDbName) .verifyResult(incrementalDump.lastReplicationId) -.run("SHOW FUNCTIONS LIKE '*testfunctionanother*'") +.run("SHOW FUNCTIONS LIKE '%testfunctionanother%'") .verifyResult(null); } @@ -223,7 +223,7 @@ public class TestReplicationScenariosAcrossInstances extends BaseReplicationAcro WarehouseInstance.Tuple bootStrapDump = primary.dump(primaryDbName, null); replica.load(replicatedDbName, bootStrapDump.dumpLocation) -.run("SHOW
[hive] branch master updated (967a1cc -> a3ee8d0)
This is an automated email from the ASF dual-hosted git repository. kgyrtkirk pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/hive.git. from 967a1cc HIVE-20854 Sensible Defaults: Hive's Zookeeper heartbeat interval is 20 minutes, change to 2 (Gopal V via Alan Gates) new bc40efd HIVE-21896: SHOW FUNCTIONS / SHOW FUNCTIONS LIKE - clarify (Miklos Gergely via Zoltan Haindrich) new a3ee8d0 HIVE-21914: Move Function and Macro related DDL operations into the DDL framework (Miklos Gergely via Zoltan Haindrich) The 2 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../TestReplicationScenariosAcrossInstances.java | 24 +- .../hive/llap/daemon/impl/FunctionLocalizer.java | 4 +- .../apache/hadoop/hive/ql/ddl/DDLOperation.java| 3 + .../{plan => ddl/function}/CreateFunctionDesc.java | 66 ++-- .../ql/ddl/function/CreateFunctionOperation.java | 217 +++ .../hive/ql/ddl/function/CreateMacroDesc.java | 81 + .../function/CreateMacroOperation.java}| 38 +- .../{plan => ddl/function}/DropFunctionDesc.java | 51 +-- .../ql/ddl/function/DropFunctionOperation.java | 107 ++ .../ql/{plan => ddl/function}/DropMacroDesc.java | 28 +- .../function/DropMacroOperation.java} | 42 +-- .../function/ReloadFunctionsDesc.java} | 13 +- .../function/ReloadFunctionsOperation.java}| 43 +-- .../hive/ql/ddl/function/ShowFunctionsDesc.java| 6 - .../ql/ddl/function/ShowFunctionsOperation.java| 7 +- .../table/storage/AlterTableSetSerdeOperation.java | 5 +- .../hadoop/hive/ql/exec/FunctionRegistry.java | 2 +- .../apache/hadoop/hive/ql/exec/FunctionTask.java | 399 - .../apache/hadoop/hive/ql/exec/FunctionUtils.java | 55 ++- .../org/apache/hadoop/hive/ql/exec/Registry.java | 4 +- .../apache/hadoop/hive/ql/exec/TaskFactory.java| 3 - .../org/apache/hadoop/hive/ql/metadata/Hive.java | 3 +- .../hive/ql/optimizer/QueryPlanPostProcessor.java | 2 - .../hadoop/hive/ql/parse/DDLSemanticAnalyzer.java | 8 +- .../hive/ql/parse/FunctionSemanticAnalyzer.java| 18 +- .../org/apache/hadoop/hive/ql/parse/HiveParser.g | 13 +- .../hive/ql/parse/MacroSemanticAnalyzer.java | 12 +- .../hive/ql/parse/SemanticAnalyzerFactory.java | 4 +- .../repl/load/message/CreateFunctionHandler.java | 12 +- .../repl/load/message/DropFunctionHandler.java | 9 +- .../hadoop/hive/ql/plan/CreateMacroDesc.java | 73 .../apache/hadoop/hive/ql/plan/FunctionWork.java | 93 - .../hive/ql/parse/TestMacroSemanticAnalyzer.java | 3 + .../hadoop/hive/ql/plan/TestCreateMacroDesc.java | 8 +- .../hadoop/hive/ql/plan/TestDropMacroDesc.java | 3 +- .../queries/clientnegative/show_functions_wrong.q | 2 + ql/src/test/queries/clientpositive/create_func1.q | 19 +- .../clientpositive/show_describe_func_quotes.q | 4 +- .../test/queries/clientpositive/show_functions.q | 20 +- .../create_function_nonexistent_class.q.out| 2 +- .../create_function_nonudf_class.q.out | 2 +- .../clientnegative/create_unknown_genericudf.q.out | 2 +- .../clientnegative/create_unknown_udf_udaf.q.out | 2 +- .../test/results/clientnegative/ivyDownload.q.out | 2 +- .../clientnegative/show_functions_wrong.q.out | 1 + .../udf_function_does_not_implement_udf.q.out | 2 +- .../clientnegative/udf_local_resource.q.out| 2 +- .../clientnegative/udf_nonexistent_resource.q.out | 2 +- .../test/results/clientpositive/create_func1.q.out | 110 +- .../clientpositive/create_genericudaf.q.out| 4 + .../results/clientpositive/create_genericudf.q.out | 4 + .../test/results/clientpositive/create_udaf.q.out | 4 + ql/src/test/results/clientpositive/drop_udf.q.out | 3 + .../clientpositive/show_describe_func_quotes.q.out | 8 +- .../results/clientpositive/show_functions.q.out| 347 -- .../clientpositive/tez/explainanalyze_3.q.out | 2 + .../results/clientpositive/tez/explainuser_3.q.out | 2 + .../clientpositive/udf_compare_java_string.q.out | 4 + .../clientpositive/udf_logic_java_boolean.q.out| 4 + .../results/clientpositive/udf_testlength.q.out| 4 + .../results/clientpositive/udf_testlength2.q.out | 4 + 61 files changed, 1151 insertions(+), 870 deletions(-) rename ql/src/java/org/apache/hadoop/hive/ql/{plan => ddl/function}/CreateFunctionDesc.java (58%) create mode 100644 ql/src/java/org/apache/hadoop/hive/ql/ddl/function/CreateFunctionOperation.java create mode 100644 ql/src/java/org/apache/hadoop/hive/ql/ddl/function/CreateMacroDesc.java copy