hive git commit: HIVE-20451 Metastore client and server tarball issues (Alexander Kolbasov via Alan Gates)
Repository: hive
Updated Branches:
refs/heads/master 189d3fec2 -> 57f40f71f
HIVE-20451 Metastore client and server tarball issues (Alexander Kolbasov via
Alan Gates)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/57f40f71
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/57f40f71
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/57f40f71
Branch: refs/heads/master
Commit: 57f40f71ff9275458e743aad0ca460dff2291a4a
Parents: 189d3fe
Author: Alan Gates
Authored: Fri Aug 31 14:03:09 2018 -0700
Committer: Alan Gates
Committed: Fri Aug 31 14:03:09 2018 -0700
--
.../tools/TestSchemaToolCatalogOps.java | 7 ++
standalone-metastore/metastore-common/pom.xml | 36 +++
.../metastore-common/src/assembly/bin.xml | 108 ---
.../metastore-common/src/assembly/src.xml | 53 -
standalone-metastore/metastore-server/pom.xml | 4 +-
.../metastore-server/src/assembly/src.xml | 53 -
standalone-metastore/metastore-tools/pom.xml| 16 +++
standalone-metastore/pom.xml| 32 +-
standalone-metastore/src/assembly/src.xml | 55 ++
9 files changed, 124 insertions(+), 240 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hive/blob/57f40f71/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/tools/TestSchemaToolCatalogOps.java
--
diff --git
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/tools/TestSchemaToolCatalogOps.java
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/tools/TestSchemaToolCatalogOps.java
index 42eb979..43c0b96 100644
---
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/tools/TestSchemaToolCatalogOps.java
+++
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/tools/TestSchemaToolCatalogOps.java
@@ -44,6 +44,7 @@ import org.junit.Test;
import java.io.File;
import java.io.IOException;
+import java.io.PrintStream;
import java.util.Collections;
import java.util.HashSet;
import java.util.Set;
@@ -56,6 +57,8 @@ public class TestSchemaToolCatalogOps {
private static HiveConf conf;
private IMetaStoreClient client;
private static String testMetastoreDB;
+ private static PrintStream errStream;
+ private static PrintStream outStream;
private static String argsBase;
@BeforeClass
@@ -76,6 +79,8 @@ public class TestSchemaToolCatalogOps {
String passWord = MetastoreConf.getPassword(conf,
MetastoreConf.ConfVars.PWD);
schemaTool.setUserName(userName);
schemaTool.setPassWord(passWord);
+errStream = System.err;
+outStream = System.out;
argsBase = "-dbType derby -userName " + userName + " -passWord " +
passWord + " ";
execute(new SchemaToolTaskInit(), "-initSchema"); // Pre-install the
database so all the tables are there.
@@ -87,6 +92,8 @@ public class TestSchemaToolCatalogOps {
if (metaStoreDir.exists()) {
FileUtils.forceDeleteOnExit(metaStoreDir);
}
+System.setOut(outStream);
+System.setErr(errStream);
}
@Before
http://git-wip-us.apache.org/repos/asf/hive/blob/57f40f71/standalone-metastore/metastore-common/pom.xml
--
diff --git a/standalone-metastore/metastore-common/pom.xml
b/standalone-metastore/metastore-common/pom.xml
index 334adf8..b05d434 100644
--- a/standalone-metastore/metastore-common/pom.xml
+++ b/standalone-metastore/metastore-common/pom.xml
@@ -446,28 +446,6 @@
org.apache.maven.plugins
-maven-assembly-plugin
-${maven.assembly.plugin.version}
-
-
-assemble
-package
-
- single
-
-
- apache-hive-metastore-${project.version}
-
-src/assembly/bin.xml
-src/assembly/src.xml
-
- gnu
-
-
-
-
-
-org.apache.maven.plugins
maven-enforcer-plugin
@@ -618,6 +596,20 @@
${basedir}/src/main/java
+
+
+org.apache.maven.plugins
+maven-assembly-plugin
+
+
+assemble
+none
+
+ single
+
+
+
+
http://git-wip-us.apache.org/repos/asf/hive/blob/57f40f71/standalone-metastore/metastore-common/src/assembly/bin.xml
--
diff --git a/standalone-metastore/metastore-common/src/assembly/bin.xml
b/standalone-metastore/metastore-common/src/assembly/bin.xml
deleted file mode
hive git commit: HIVE-14162: Allow disabling of long running job on Hive On Spark On YARN (Sahil Takiar, reviewed by Adam Szita)
Repository: hive
Updated Branches:
refs/heads/master 219538701 -> 189d3fec2
HIVE-14162: Allow disabling of long running job on Hive On Spark On YARN (Sahil
Takiar, reviewed by Adam Szita)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/189d3fec
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/189d3fec
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/189d3fec
Branch: refs/heads/master
Commit: 189d3fec25dfb94b209b1a34c1be674ce9d85bc5
Parents: 2195387
Author: Sahil Takiar
Authored: Mon Jul 16 10:26:21 2018 -0500
Committer: Sahil Takiar
Committed: Fri Aug 31 15:42:57 2018 -0500
--
.../org/apache/hadoop/hive/conf/HiveConf.java | 6 +
.../ql/exec/spark/TestSparkSessionTimeout.java | 145 +++
.../java/org/apache/hadoop/hive/ql/Driver.java | 10 +
.../ql/exec/spark/session/SparkSession.java | 27 ++
.../ql/exec/spark/session/SparkSessionImpl.java | 246 ++-
.../spark/session/SparkSessionManagerImpl.java | 63 +++--
6 files changed, 423 insertions(+), 74 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hive/blob/189d3fec/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
--
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 8c39de3..40ea3ac 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -4311,6 +4311,12 @@ public class HiveConf extends Configuration {
"specified (default) then the spark-submit shell script is used to
launch the Spark " +
"app. If " + HIVE_SPARK_LAUNCHER_CLIENT + " is specified then Spark's
" +
"InProcessLauncher is used to programmatically launch the app."),
+SPARK_SESSION_TIMEOUT("hive.spark.session.timeout", "30m", new
TimeValidator(TimeUnit.MINUTES,
+30L, true, null, true), "Amount of time the Spark Remote Driver
should wait for " +
+" a Spark job to be submitted before shutting down. Minimum value
is 30 minutes"),
+SPARK_SESSION_TIMEOUT_PERIOD("hive.spark.session.timeout.period", "60s",
+new TimeValidator(TimeUnit.SECONDS, 60L, true, null, true),
+"How frequently to check for idle Spark sessions. Minimum value is
60 seconds."),
NWAYJOINREORDER("hive.reorder.nway.joins", true,
"Runs reordering of tables within single n-way join (i.e.: picks
streamtable)"),
HIVE_MERGE_NWAY_JOINS("hive.merge.nway.joins", true,
http://git-wip-us.apache.org/repos/asf/hive/blob/189d3fec/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/exec/spark/TestSparkSessionTimeout.java
--
diff --git
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/exec/spark/TestSparkSessionTimeout.java
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/exec/spark/TestSparkSessionTimeout.java
new file mode 100644
index 000..c887297
--- /dev/null
+++
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/exec/spark/TestSparkSessionTimeout.java
@@ -0,0 +1,145 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.exec.spark;
+
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.ql.Driver;
+import org.apache.hadoop.hive.ql.QueryState;
+import org.apache.hadoop.hive.ql.exec.spark.session.SparkSession;
+import org.apache.hadoop.hive.ql.exec.spark.session.SparkSessionManagerImpl;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.session.SessionState;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.io.File;
+import java.net.MalformedURLException;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.UUID;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
[1/2] hive git commit: HIVE-20225: SerDe to support Teradata Binary Format (Lu Li via cws)
Repository: hive Updated Branches: refs/heads/branch-2 bd32deb44 -> 8b16ad0f5 http://git-wip-us.apache.org/repos/asf/hive/blob/8b16ad0f/src/test/queries/clientpositive/test_teradatabinaryfile.q -- diff --git a/src/test/queries/clientpositive/test_teradatabinaryfile.q b/src/test/queries/clientpositive/test_teradatabinaryfile.q new file mode 100644 index 000..33ab677 --- /dev/null +++ b/src/test/queries/clientpositive/test_teradatabinaryfile.q @@ -0,0 +1,123 @@ +DROP TABLE if exists teradata_binary_table_64kb; +DROP TABLE if exists teradata_binary_table_1mb; +DROP TABLE if exists teradata_binary_table_64kb_insert; +DROP TABLE if exists teradata_binary_table_1mb_insert; + + +CREATE TABLE `teradata_binary_table_64kb`( + `test_tinyint` tinyint, + `test_smallint` smallint, + `test_int` int, + `test_bigint` bigint, + `test_double` double, + `test_decimal` decimal(15,2), + `test_date` date, + `test_timestamp` timestamp, + `test_char` char(1), + `test_varchar` varchar(40), + `test_binary` binary + ) +ROW FORMAT SERDE + 'org.apache.hadoop.hive.serde2.teradata.TeradataBinarySerde' +STORED AS INPUTFORMAT + 'org.apache.hadoop.hive.ql.io.TeradataBinaryFileInputFormat' +OUTPUTFORMAT + 'org.apache.hadoop.hive.ql.io.TeradataBinaryFileOutputFormat' +TBLPROPERTIES ( + 'teradata.timestamp.precision'='0', + 'teradata.char.charset'='LATIN', + 'teradata.row.length'='64KB' +); + +CREATE TABLE `teradata_binary_table_1mb`( + `test_tinyint` tinyint, + `test_smallint` smallint, + `test_int` int, + `test_bigint` bigint, + `test_double` double, + `test_decimal` decimal(15,2), + `test_date` date, + `test_timestamp` timestamp, + `test_char` char(1), + `test_varchar` varchar(40), + `test_binary` binary + ) +ROW FORMAT SERDE + 'org.apache.hadoop.hive.serde2.teradata.TeradataBinarySerde' +STORED AS INPUTFORMAT + 'org.apache.hadoop.hive.ql.io.TeradataBinaryFileInputFormat' +OUTPUTFORMAT + 'org.apache.hadoop.hive.ql.io.TeradataBinaryFileOutputFormat' +TBLPROPERTIES ( + 'teradata.timestamp.precision'='6', + 'teradata.char.charset'='UNICODE', + 'teradata.row.length'='1MB' +); + +CREATE TABLE `teradata_binary_table_64kb_insert`( + `test_tinyint` tinyint, + `test_decimal` decimal(15,2), + `test_date` date, + `test_timestamp` timestamp + ) +ROW FORMAT SERDE + 'org.apache.hadoop.hive.serde2.teradata.TeradataBinarySerde' +STORED AS INPUTFORMAT + 'org.apache.hadoop.hive.ql.io.TeradataBinaryFileInputFormat' +OUTPUTFORMAT + 'org.apache.hadoop.hive.ql.io.TeradataBinaryFileOutputFormat' +TBLPROPERTIES ( + 'teradata.timestamp.precision'='0', + 'teradata.char.charset'='LATIN', + 'teradata.row.length'='64KB' +); + +CREATE TABLE `teradata_binary_table_1mb_insert`( + `test_tinyint` tinyint, + `test_int` int + ) +ROW FORMAT SERDE + 'org.apache.hadoop.hive.serde2.teradata.TeradataBinarySerde' +STORED AS INPUTFORMAT + 'org.apache.hadoop.hive.ql.io.TeradataBinaryFileInputFormat' +OUTPUTFORMAT + 'org.apache.hadoop.hive.ql.io.TeradataBinaryFileOutputFormat' +TBLPROPERTIES ( + 'teradata.timestamp.precision'='6', + 'teradata.char.charset'='UNICODE', + 'teradata.row.length'='1MB' +); + +LOAD DATA LOCAL INPATH '../../data/files/teradata_binary_file/teradata_binary_table.deflate' OVERWRITE INTO TABLE teradata_binary_table_64kb; +LOAD DATA LOCAL INPATH '../../data/files/teradata_binary_file/td_data_with_1mb_rowsize.teradata.gz' OVERWRITE INTO TABLE teradata_binary_table_1mb; + +SELECT * from teradata_binary_table_64kb; +SELECT * from teradata_binary_table_1mb; + +SELECT COUNT(*) FROM teradata_binary_table_64kb; +SELECT COUNT(*) FROM teradata_binary_table_1mb; + +SELECT max(date_format(test_timestamp, 'y')) FROM teradata_binary_table_64kb; +SELECT max(date_format(test_date, 'y')) FROM teradata_binary_table_64kb; +SELECT max(Floor(test_decimal)) FROM teradata_binary_table_64kb; + +SELECT max(date_format(test_timestamp, 'y')) FROM teradata_binary_table_1mb; +SELECT max(date_format(test_date, 'y')) FROM teradata_binary_table_1mb; +SELECT max(Floor(test_decimal)) FROM teradata_binary_table_1mb; + +SELECT test_tinyint, MAX(test_decimal) FROM teradata_binary_table_64kb GROUP BY test_tinyint; +SELECT test_tinyint, MAX(test_decimal) FROM teradata_binary_table_1mb GROUP BY test_tinyint; + +INSERT OVERWRITE TABLE teradata_binary_table_64kb_insert +SELECT test_tinyint, test_decimal, test_date, test_timestamp FROM teradata_binary_table_64kb; + +INSERT OVERWRITE TABLE teradata_binary_table_1mb_insert +SELECT 1, 15; + +DESC FORMATTED teradata_binary_table_64kb_insert; +DESC FORMATTED teradata_binary_table_1mb_insert; + +DROP TABLE if exists teradata_binary_table_64kb; +DROP TABLE if exists teradata_binary_table_1mb; +DROP TABLE if exists teradata_binary_table_64kb_insert; +DROP TABLE if exists teradata_binary_table_1mb_insert; http://git-wip-us.apache.org/repos/asf/hive/blob/8b16ad0f/src/test/results/clientpositive/test_teradatabinaryfile.q.out
[2/2] hive git commit: HIVE-20225: SerDe to support Teradata Binary Format (Lu Li via cws)
HIVE-20225: SerDe to support Teradata Binary Format (Lu Li via cws) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/8b16ad0f Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/8b16ad0f Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/8b16ad0f Branch: refs/heads/branch-2 Commit: 8b16ad0f54d3bcf56362118f31e25a792ad7291e Parents: bd32deb Author: Daniel Dai Authored: Fri Aug 31 12:36:23 2018 -0700 Committer: Daniel Dai Committed: Fri Aug 31 12:36:23 2018 -0700 -- .../td_data_with_1mb_rowsize.teradata.gz| Bin 0 -> 616 bytes .../teradata_binary_table.deflate | Bin 0 -> 1329 bytes .../ql/io/TeradataBinaryFileInputFormat.java| 66 ++ .../ql/io/TeradataBinaryFileOutputFormat.java | 112 .../hive/ql/io/TeradataBinaryRecordReader.java | 280 + .../teradata/TeradataBinaryDataInputStream.java | 200 +++ .../TeradataBinaryDataOutputStream.java | 270 + .../serde2/teradata/TeradataBinarySerde.java| 597 +++ .../TestTeradataBinarySerdeForDate.java | 72 +++ .../TestTeradataBinarySerdeForDecimal.java | 106 .../TestTeradataBinarySerdeForTimeStamp.java| 96 +++ .../TestTeradataBinarySerdeGeneral.java | 126 .../clientpositive/test_teradatabinaryfile.q| 123 .../test_teradatabinaryfile.q.out | 537 + 14 files changed, 2585 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hive/blob/8b16ad0f/data/files/teradata_binary_file/td_data_with_1mb_rowsize.teradata.gz -- diff --git a/data/files/teradata_binary_file/td_data_with_1mb_rowsize.teradata.gz b/data/files/teradata_binary_file/td_data_with_1mb_rowsize.teradata.gz new file mode 100644 index 000..7319e3c Binary files /dev/null and b/data/files/teradata_binary_file/td_data_with_1mb_rowsize.teradata.gz differ http://git-wip-us.apache.org/repos/asf/hive/blob/8b16ad0f/data/files/teradata_binary_file/teradata_binary_table.deflate -- diff --git a/data/files/teradata_binary_file/teradata_binary_table.deflate b/data/files/teradata_binary_file/teradata_binary_table.deflate new file mode 100644 index 000..fd53dde Binary files /dev/null and b/data/files/teradata_binary_file/teradata_binary_table.deflate differ http://git-wip-us.apache.org/repos/asf/hive/blob/8b16ad0f/src/java/org/apache/hadoop/hive/ql/io/TeradataBinaryFileInputFormat.java -- diff --git a/src/java/org/apache/hadoop/hive/ql/io/TeradataBinaryFileInputFormat.java b/src/java/org/apache/hadoop/hive/ql/io/TeradataBinaryFileInputFormat.java new file mode 100644 index 000..bed87c5 --- /dev/null +++ b/src/java/org/apache/hadoop/hive/ql/io/TeradataBinaryFileInputFormat.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.io; + +import java.io.IOException; + +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.BytesWritable; +import org.apache.hadoop.io.NullWritable; +import org.apache.hadoop.mapred.FileInputFormat; +import org.apache.hadoop.mapred.FileSplit; +import org.apache.hadoop.mapred.InputSplit; +import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.mapred.RecordReader; +import org.apache.hadoop.mapred.Reporter; + +/** + * https://cwiki.apache.org/confluence/display/Hive/TeradataBinarySerde. + * FileInputFormat for Teradata binary files. + * + * In the Teradata Binary File, each record constructs as below: + * The first 2 bytes represents the length of the bytes next for this record. + * Then the null bitmap whose length is depended on the number of fields is followed. + * Then each field of the record is serialized into bytes - the serialization strategy is decided by the type of field. + * At last, there is one byte (0x0a) in the end of the
hive git commit: HIVE-20225: SerDe to support Teradata Binary Format (Lu Li via cws)
Repository: hive Updated Branches: refs/heads/master a77e0468f -> 219538701 HIVE-20225: SerDe to support Teradata Binary Format (Lu Li via cws) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/21953870 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/21953870 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/21953870 Branch: refs/heads/master Commit: 219538701967fc1f5386c2f107fd207988250f2e Parents: a77e046 Author: Daniel Dai Authored: Fri Aug 31 12:19:29 2018 -0700 Committer: Daniel Dai Committed: Fri Aug 31 12:19:29 2018 -0700 -- .../td_data_with_1mb_rowsize.teradata.gz| Bin 0 -> 616 bytes .../teradata_binary_table.deflate | Bin 0 -> 1329 bytes 2 files changed, 0 insertions(+), 0 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hive/blob/21953870/data/files/teradata_binary_file/td_data_with_1mb_rowsize.teradata.gz -- diff --git a/data/files/teradata_binary_file/td_data_with_1mb_rowsize.teradata.gz b/data/files/teradata_binary_file/td_data_with_1mb_rowsize.teradata.gz new file mode 100644 index 000..7319e3c Binary files /dev/null and b/data/files/teradata_binary_file/td_data_with_1mb_rowsize.teradata.gz differ http://git-wip-us.apache.org/repos/asf/hive/blob/21953870/data/files/teradata_binary_file/teradata_binary_table.deflate -- diff --git a/data/files/teradata_binary_file/teradata_binary_table.deflate b/data/files/teradata_binary_file/teradata_binary_table.deflate new file mode 100644 index 000..fd53dde Binary files /dev/null and b/data/files/teradata_binary_file/teradata_binary_table.deflate differ
