hive git commit: HIVE-19814: RPC Server port is always random for spark (Bharathkrishna Guruvayoor Murali, reviewed by Sahil Takiar)
Repository: hive
Updated Branches:
refs/heads/master 84e5b9391 -> a3b7a2452
HIVE-19814: RPC Server port is always random for spark (Bharathkrishna
Guruvayoor Murali, reviewed by Sahil Takiar)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/a3b7a245
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/a3b7a245
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/a3b7a245
Branch: refs/heads/master
Commit: a3b7a2452bacf6d7eeeb42bd9dd68109c90e27a2
Parents: 84e5b93
Author: Bharathkrishna Guruvayoor Murali
Authored: Wed Sep 12 17:50:29 2018 -0700
Committer: Sahil Takiar
Committed: Wed Sep 12 17:50:54 2018 -0700
--
.../ql/exec/spark/HiveSparkClientFactory.java | 8 -
.../session/TestSparkSessionManagerImpl.java| 35
.../hive/spark/client/SparkClientFactory.java | 6
.../hive/spark/client/rpc/RpcConfiguration.java | 17 +-
4 files changed, 57 insertions(+), 9 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hive/blob/a3b7a245/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HiveSparkClientFactory.java
--
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HiveSparkClientFactory.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HiveSparkClientFactory.java
index 0aae0d8..a49e72d 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HiveSparkClientFactory.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HiveSparkClientFactory.java
@@ -32,6 +32,7 @@ import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.hive.common.LogUtils;
import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hive.spark.client.SparkClientUtilities;
+import org.apache.hive.spark.client.rpc.RpcConfiguration;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.hbase.HBaseConfiguration;
@@ -198,7 +199,12 @@ public class HiveSparkClientFactory {
LOG.debug(String.format(
"Pass Oozie configuration (%s -> %s).", propertyName,
LogUtils.maskIfPassword(propertyName,value)));
}
-
+ if (RpcConfiguration.HIVE_SPARK_RSC_CONFIGS.contains(propertyName)) {
+String value = RpcConfiguration.getValue(hiveConf, propertyName);
+sparkConf.put(propertyName, value);
+LOG.debug(String.format("load RPC property from hive configuration (%s
-> %s).", propertyName,
+LogUtils.maskIfPassword(propertyName, value)));
+ }
}
final boolean optShuffleSerDe = hiveConf.getBoolVar(
http://git-wip-us.apache.org/repos/asf/hive/blob/a3b7a245/ql/src/test/org/apache/hadoop/hive/ql/exec/spark/session/TestSparkSessionManagerImpl.java
--
diff --git
a/ql/src/test/org/apache/hadoop/hive/ql/exec/spark/session/TestSparkSessionManagerImpl.java
b/ql/src/test/org/apache/hadoop/hive/ql/exec/spark/session/TestSparkSessionManagerImpl.java
index 853e4f4..3882b58 100644
---
a/ql/src/test/org/apache/hadoop/hive/ql/exec/spark/session/TestSparkSessionManagerImpl.java
+++
b/ql/src/test/org/apache/hadoop/hive/ql/exec/spark/session/TestSparkSessionManagerImpl.java
@@ -22,6 +22,8 @@ import org.apache.hadoop.hive.ql.ErrorMsg;
import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hadoop.util.StringUtils;
+import org.apache.hive.spark.client.SparkClientFactory;
+import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -31,6 +33,7 @@ import org.slf4j.LoggerFactory;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List;
+import java.util.Map;
import java.util.Random;
import java.util.concurrent.TimeoutException;
@@ -214,6 +217,38 @@ public class TestSparkSessionManagerImpl {
assertEquals("0", ss.getSparkSession().getSessionId());
}
+ @Test
+ public void testConfigsForInitialization() {
+//Test to make sure that configs listed in
RpcConfiguration.HIVE_SPARK_RSC_CONFIGS which are passed
+// through HiveConf are included in the Spark configuration.
+HiveConf hiveConf = getHiveConf();
+hiveConf.setVar(HiveConf.ConfVars.SPARK_RPC_SERVER_PORT,
"49152-49222,49223,49224-49333");
+hiveConf.setVar(HiveConf.ConfVars.SPARK_RPC_SERVER_ADDRESS,
"test-rpc-server-address");
+Map sparkConf =
HiveSparkClientFactory.initiateSparkConf(hiveConf, null);
+assertEquals("49152-49222,49223,49224-49333",
sparkConf.get(HiveConf.ConfVars.SPARK_RPC_SERVER_PORT.varname));
+assertEquals("test-rpc-server-address",
sparkConf.get(HiveConf.ConfVars.SPARK_RPC_SERVER_ADDRESS.varname));
+ }
+
+ @Test
+ public void testServerPortAssignment() throws Exception {
+HiveConf conf = getHiveConf();
+conf.setVar
hive git commit: HIVE-20412: NPE in HiveMetaHook (Jason Dere, reviewed by Sergey Shelukhin)
Repository: hive
Updated Branches:
refs/heads/master f4380f3b5 -> 84e5b9391
HIVE-20412: NPE in HiveMetaHook (Jason Dere, reviewed by Sergey Shelukhin)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/84e5b939
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/84e5b939
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/84e5b939
Branch: refs/heads/master
Commit: 84e5b939190fb18fe0cac232d7e3edf20fc41652
Parents: f4380f3
Author: Jason Dere
Authored: Wed Sep 12 12:29:37 2018 -0700
Committer: Jason Dere
Committed: Wed Sep 12 12:29:37 2018 -0700
--
.../main/java/org/apache/hadoop/hive/metastore/HiveMetaHook.java | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hive/blob/84e5b939/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaHook.java
--
diff --git
a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaHook.java
b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaHook.java
index ad596d1..3a827f7 100644
---
a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaHook.java
+++
b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaHook.java
@@ -110,7 +110,8 @@ public interface HiveMetaHook {
* @param table new table definition
*/
public default void preAlterTable(Table table, EnvironmentContext context)
throws MetaException {
-String alterOpType = context == null ? null :
context.getProperties().get(ALTER_TABLE_OPERATION_TYPE);
+String alterOpType = (context == null || context.getProperties() == null) ?
+null : context.getProperties().get(ALTER_TABLE_OPERATION_TYPE);
// By default allow only ADDPROPS and DROPPROPS.
// alterOpType is null in case of stats update.
if (alterOpType != null && !allowedAlterTypes.contains(alterOpType)){
hive git commit: HIVE-19552: Enable TestMiniDruidKafkaCliDriver#druidkafkamini_basic.q (Nishant Bangarwa, reviewed by Jesus Camacho Rodriguez)
Repository: hive
Updated Branches:
refs/heads/branch-3 dbc5b176f -> f1d8c3f61
HIVE-19552: Enable TestMiniDruidKafkaCliDriver#druidkafkamini_basic.q (Nishant
Bangarwa, reviewed by Jesus Camacho Rodriguez)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/f1d8c3f6
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/f1d8c3f6
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/f1d8c3f6
Branch: refs/heads/branch-3
Commit: f1d8c3f614a15ebe691319e989dee906df15edd8
Parents: dbc5b17
Author: Nishant Bangarwa
Authored: Wed Sep 12 09:58:45 2018 -0700
Committer: Jesus Camacho Rodriguez
Committed: Wed Sep 12 10:00:23 2018 -0700
--
.../src/main/java/org/apache/hadoop/hive/ql/QOutProcessor.java | 3 ++-
.../results/clientpositive/druid/druidkafkamini_basic.q.out| 6 ++
2 files changed, 4 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hive/blob/f1d8c3f6/itests/util/src/main/java/org/apache/hadoop/hive/ql/QOutProcessor.java
--
diff --git
a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QOutProcessor.java
b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QOutProcessor.java
index 359f027..400eec7 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QOutProcessor.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QOutProcessor.java
@@ -121,7 +121,8 @@ public class QOutProcessor {
".*at com\\.zaxxer.*",
"org\\.apache\\.hadoop\\.hive\\.metastore\\.model\\.MConstraint@([0-9]|[a-z])*",
"^Repair: Added partition to metastore.*",
- "^Repair: Dropped partition from metastore.*"
+ "^latestOffsets.*",
+ "^minimumLag.*"
});
public QOutProcessor(FsType fsType) {
http://git-wip-us.apache.org/repos/asf/hive/blob/f1d8c3f6/ql/src/test/results/clientpositive/druid/druidkafkamini_basic.q.out
--
diff --git
a/ql/src/test/results/clientpositive/druid/druidkafkamini_basic.q.out
b/ql/src/test/results/clientpositive/druid/druidkafkamini_basic.q.out
index ffd9279..b664e8c 100644
--- a/ql/src/test/results/clientpositive/druid/druidkafkamini_basic.q.out
+++ b/ql/src/test/results/clientpositive/druid/druidkafkamini_basic.q.out
@@ -72,8 +72,7 @@ Druid Storage Handler Runtime Status for
default.druid_kafka_test
kafkaPartitions=1
activeTasks=[]
publishingTasks=[]
-latestOffsets={0=10}
-minimumLag={0=0}
+ A masked pattern was here
aggregateLag=0
A masked pattern was here
PREHOOK: query: Select count(*) FROM druid_kafka_test
@@ -142,8 +141,7 @@ Druid Storage Handler Runtime Status for
default.druid_kafka_test
kafkaPartitions=1
activeTasks=[]
publishingTasks=[]
-latestOffsets={0=10}
-minimumLag={0=0}
+ A masked pattern was here
aggregateLag=0
A masked pattern was here
PREHOOK: query: Select count(*) FROM druid_kafka_test
hive git commit: HIVE-19552: Enable TestMiniDruidKafkaCliDriver#druidkafkamini_basic.q (Nishant Bangarwa, reviewed by Jesus Camacho Rodriguez)
Repository: hive
Updated Branches:
refs/heads/master 294665acf -> f4380f3b5
HIVE-19552: Enable TestMiniDruidKafkaCliDriver#druidkafkamini_basic.q (Nishant
Bangarwa, reviewed by Jesus Camacho Rodriguez)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/f4380f3b
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/f4380f3b
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/f4380f3b
Branch: refs/heads/master
Commit: f4380f3b5ea7997185ceefb4315bc2e325c99c9f
Parents: 294665a
Author: Nishant Bangarwa
Authored: Wed Sep 12 09:58:45 2018 -0700
Committer: Jesus Camacho Rodriguez
Committed: Wed Sep 12 09:58:59 2018 -0700
--
.../java/org/apache/hadoop/hive/cli/control/CliConfigs.java| 1 -
.../src/main/java/org/apache/hadoop/hive/ql/QOutProcessor.java | 3 ++-
.../results/clientpositive/druid/druidkafkamini_basic.q.out| 6 ++
3 files changed, 4 insertions(+), 6 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hive/blob/f4380f3b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java
--
diff --git
a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java
b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java
index 3466bac..a3dcf98 100644
---
a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java
+++
b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java
@@ -202,7 +202,6 @@ public class CliConfigs {
setQueryDir("ql/src/test/queries/clientpositive");
includesFrom(testConfigProps, "druid.kafka.query.files");
-excludeQuery("druidkafkamini_basic.q"); // HIVE-19509
setResultsDir("ql/src/test/results/clientpositive/druid");
setLogDir("itests/qtest/target/tmp/log");
http://git-wip-us.apache.org/repos/asf/hive/blob/f4380f3b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QOutProcessor.java
--
diff --git
a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QOutProcessor.java
b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QOutProcessor.java
index 1e4cddd..a17c508 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QOutProcessor.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QOutProcessor.java
@@ -121,7 +121,8 @@ public class QOutProcessor {
".*at com\\.zaxxer.*",
"org\\.apache\\.hadoop\\.hive\\.metastore\\.model\\.MConstraint@([0-9]|[a-z])*",
"^Repair: Added partition to metastore.*",
- "^Repair: Dropped partition from metastore.*"
+ "^latestOffsets.*",
+ "^minimumLag.*"
});
public QOutProcessor(FsType fsType) {
http://git-wip-us.apache.org/repos/asf/hive/blob/f4380f3b/ql/src/test/results/clientpositive/druid/druidkafkamini_basic.q.out
--
diff --git
a/ql/src/test/results/clientpositive/druid/druidkafkamini_basic.q.out
b/ql/src/test/results/clientpositive/druid/druidkafkamini_basic.q.out
index c4b7b6e..619b95e 100644
--- a/ql/src/test/results/clientpositive/druid/druidkafkamini_basic.q.out
+++ b/ql/src/test/results/clientpositive/druid/druidkafkamini_basic.q.out
@@ -72,8 +72,7 @@ Druid Storage Handler Runtime Status for
default.druid_kafka_test
kafkaPartitions=1
activeTasks=[]
publishingTasks=[]
-latestOffsets={0=10}
-minimumLag={0=0}
+ A masked pattern was here
aggregateLag=0
A masked pattern was here
PREHOOK: query: Select count(*) FROM druid_kafka_test
@@ -142,8 +141,7 @@ Druid Storage Handler Runtime Status for
default.druid_kafka_test
kafkaPartitions=1
activeTasks=[]
publishingTasks=[]
-latestOffsets={0=10}
-minimumLag={0=0}
+ A masked pattern was here
aggregateLag=0
A masked pattern was here
PREHOOK: query: Select count(*) FROM druid_kafka_test
hive git commit: HIVE-20539 : Remove dependency on com.metamx.java-util (Nishant Bangarwa via Ashutosh Chauhan)
Repository: hive
Updated Branches:
refs/heads/master f18842e73 -> 294665acf
HIVE-20539 : Remove dependency on com.metamx.java-util (Nishant Bangarwa via
Ashutosh Chauhan)
Signed-off-by: Ashutosh Chauhan
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/294665ac
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/294665ac
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/294665ac
Branch: refs/heads/master
Commit: 294665acf77c02d27a7a1b5a84a6b21b1e013701
Parents: f18842e
Author: Nishant Bangarwa
Authored: Wed Sep 12 08:07:42 2018 -0700
Committer: Ashutosh Chauhan
Committed: Wed Sep 12 08:07:42 2018 -0700
--
druid-handler/pom.xml | 26
.../hadoop/hive/druid/DruidStorageHandler.java | 16 ++--
.../hive/druid/DruidStorageHandlerUtils.java| 20 +++
.../druid/io/DruidQueryBasedInputFormat.java| 2 +-
.../hive/druid/security/KerberosHttpClient.java | 8 +++---
.../druid/security/ResponseCookieHandler.java | 4 +--
.../RetryIfUnauthorizedResponseHandler.java | 4 +--
.../serde/DruidGroupByQueryRecordReader.java| 2 +-
.../druid/serde/DruidQueryRecordReader.java | 6 ++---
.../hadoop/hive/druid/serde/TestDruidSerDe.java | 4 +--
10 files changed, 33 insertions(+), 59 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hive/blob/294665ac/druid-handler/pom.xml
--
diff --git a/druid-handler/pom.xml b/druid-handler/pom.xml
index ef1231f..5b8a601 100644
--- a/druid-handler/pom.xml
+++ b/druid-handler/pom.xml
@@ -29,7 +29,6 @@
..
-1.3.2
16.0.1
@@ -87,26 +86,6 @@
${druid.guava.version}
- com.metamx
- java-util
- ${druid.metamx.util.version}
-
-
- com.fasterxml.jackson.core
- jackson-core
-
-
- com.fasterxml.jackson.core
- jackson-annotations
-
-
- com.fasterxml.jackson.core
- jackson-databind
-
-
- true
-
-
joda-time
joda-time
${joda.version}
@@ -316,10 +295,6 @@
org.apache.hive.druid.io.druid
- com.metamx
-
org.apache.hive.druid.com.metamx
-
-
io.netty
org.apache.hive.druid.io.netty
@@ -348,7 +323,6 @@
io.druid:*
io.druid.extensions:*
- com.metamx:*
io.netty:*
org.apache.calcite:*
com.fasterxml.jackson.core:*
http://git-wip-us.apache.org/repos/asf/hive/blob/294665ac/druid-handler/src/java/org/apache/hadoop/hive/druid/DruidStorageHandler.java
--
diff --git
a/druid-handler/src/java/org/apache/hadoop/hive/druid/DruidStorageHandler.java
b/druid-handler/src/java/org/apache/hadoop/hive/druid/DruidStorageHandler.java
index 9f34b7b..f21bbee 100644
---
a/druid-handler/src/java/org/apache/hadoop/hive/druid/DruidStorageHandler.java
+++
b/druid-handler/src/java/org/apache/hadoop/hive/druid/DruidStorageHandler.java
@@ -27,14 +27,6 @@ import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Sets;
-import com.metamx.common.RetryUtils;
-import com.metamx.common.lifecycle.Lifecycle;
-import com.metamx.http.client.HttpClient;
-import com.metamx.http.client.HttpClientConfig;
-import com.metamx.http.client.HttpClientInit;
-import com.metamx.http.client.Request;
-import com.metamx.http.client.response.FullResponseHandler;
-import com.metamx.http.client.response.FullResponseHolder;
import io.druid.data.input.impl.DimensionSchema;
import io.druid.data.input.impl.DimensionsSpec;
import io.druid.data.input.impl.InputRowParser;
@@ -42,6 +34,14 @@ import io.druid.data.input.impl.JSONParseSpec;
import io.druid.data.input.impl.StringInputRowParser;
import io.druid.data.input.impl.TimestampSpec;
import io.druid.java.util.common.Pair;
+import io.druid.java.util.common.RetryUtils;
+import io.druid.java.util.common.lifecycle.Lifecycle;
+import io.druid.java.util.http.client.HttpClient;
+import io.druid.java.util.http.client.HttpClientConfig;
+import io.druid.java.util.http.client.HttpClientInit;
+import io.druid.java.util.http.client.Request;
+import io.druid.java.util.http.client.response.FullResponseHandler;
+import io.druid.java.util.http.client.response.FullResponseHolder;
import io.druid.me
