[hive] branch master updated: HIVE-21283: Create Synonym mid for substr, position for locate (Mani M, reviewed by Sankar Hariappan)
This is an automated email from the ASF dual-hosted git repository.
sankarh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 2b181dc HIVE-21283: Create Synonym mid for substr, position for
locate (Mani M, reviewed by Sankar Hariappan)
2b181dc is described below
commit 2b181dc7aeccf3d92968e68be0d3a9aabd0350e2
Author: Mani M
AuthorDate: Fri Mar 22 09:46:43 2019 +0530
HIVE-21283: Create Synonym mid for substr, position for locate (Mani M,
reviewed by Sankar Hariappan)
Signed-off-by: Sankar Hariappan
---
.../hadoop/hive/ql/exec/FunctionRegistry.java | 2 +
.../org/apache/hadoop/hive/ql/udf/UDFSubstr.java | 2 +-
.../hive/ql/udf/generic/GenericUDFLocate.java | 2 +-
ql/src/test/queries/clientpositive/udf_mid.q | 3 +
ql/src/test/queries/clientpositive/udf_position.q | 44
.../results/clientpositive/show_functions.q.out| 3 +
.../test/results/clientpositive/udf_locate.q.out | 1 +
ql/src/test/results/clientpositive/udf_mid.q.out | 21
.../test/results/clientpositive/udf_position.q.out | 121 +
.../test/results/clientpositive/udf_substr.q.out | 2 +-
.../results/clientpositive/udf_substring.q.out | 2 +-
11 files changed, 199 insertions(+), 4 deletions(-)
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
index 3e4aa74..ed41bef 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
@@ -193,6 +193,7 @@ public final class FunctionRegistry {
static {
system.registerGenericUDF("concat", GenericUDFConcat.class);
+system.registerUDF("mid", UDFSubstr.class, false);
system.registerUDF("substr", UDFSubstr.class, false);
system.registerUDF("substring", UDFSubstr.class, false);
system.registerGenericUDF("substring_index",
GenericUDFSubstringIndex.class);
@@ -498,6 +499,7 @@ public final class FunctionRegistry {
system.registerGenericUDF("in_file", GenericUDFInFile.class);
system.registerGenericUDF("instr", GenericUDFInstr.class);
system.registerGenericUDF("locate", GenericUDFLocate.class);
+system.registerGenericUDF("position", GenericUDFLocate.class);
system.registerGenericUDF("elt", GenericUDFElt.class);
system.registerGenericUDF("concat_ws", GenericUDFConcatWS.class);
system.registerGenericUDF("sort_array", GenericUDFSortArray.class);
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFSubstr.java
b/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFSubstr.java
index 4136cc7..5b1964c 100755
--- a/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFSubstr.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFSubstr.java
@@ -33,7 +33,7 @@ import org.apache.hadoop.io.Text;
* UDFSubstr.
*
*/
-@Description(name = "substr,substring",
+@Description(name = "substr,substring,mid",
value = "_FUNC_(str, pos[, len]) - returns the substring of str that"
+ " starts at pos and is of length len or" +
"_FUNC_(bin, pos[, len]) - returns the slice of byte array that"
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFLocate.java
b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFLocate.java
index cebc752..a077222 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFLocate.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFLocate.java
@@ -42,7 +42,7 @@ import org.apache.hadoop.io.Text;
*
*
*/
-@Description(name = "locate",
+@Description(name = "locate,position",
value = "_FUNC_(substr, str[, pos]) - Returns the position of the first "
+ "occurance of substr in str after position pos",
extended = "Example:\n"
diff --git a/ql/src/test/queries/clientpositive/udf_mid.q
b/ql/src/test/queries/clientpositive/udf_mid.q
new file mode 100644
index 000..a2a6526
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/udf_mid.q
@@ -0,0 +1,3 @@
+-- Synonym. See udf_substr.q
+DESCRIBE FUNCTION mid;
+DESCRIBE FUNCTION EXTENDED mid;
diff --git a/ql/src/test/queries/clientpositive/udf_position.q
b/ql/src/test/queries/clientpositive/udf_position.q
new file mode 100644
index 000..232ed77
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/udf_position.q
@@ -0,0 +1,44 @@
+--! qt:dataset:src
+set hive.fetch.task.conversion=more;
+
+DESCRIBE FUNCTION position;
+DESCRIBE FUNCTION EXTENDED position;
+
+EXPLAIN
+SELECT position('abc', 'abcd'),
+ position('ccc', 'abcabc'),
+ position('23', 123),
+ position(23, 123),
+ position('abc', 'abcabc', 2),
+ position('abc', 'abcabc', '2'),
+ position(1, TRUE),
+ position(1, FALSE),
+ position(CAST('2' AS TINYINT), '12345'),
+ position('34', CAST('12345' AS SMALLINT)),
+
[hive] branch master updated: HIVE-21482: Partition discovery table property is added to non-partitioned external tables (Prasanth Jayachandran reviewed by Jason Dere)
This is an automated email from the ASF dual-hosted git repository.
prasanthj pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new b953c4e HIVE-21482: Partition discovery table property is added to
non-partitioned external tables (Prasanth Jayachandran reviewed by Jason Dere)
b953c4e is described below
commit b953c4e077e71ddf667595b6acba9d5c9972ee6a
Author: Prasanth Jayachandran
AuthorDate: Thu Mar 21 17:41:38 2019 -0700
HIVE-21482: Partition discovery table property is added to non-partitioned
external tables (Prasanth Jayachandran reviewed by Jason Dere)
---
.../test/results/positive/external_table_ppd.q.out | 1 -
.../positive/hbase_binary_storage_queries.q.out| 2 -
.../src/test/results/positive/hbase_ddl.q.out | 2 -
.../src/test/results/positive/hbase_queries.q.out | 1 -
.../src/test/results/positive/hbasestats.q.out | 5 -
.../hadoop/hive/ql/ddl/table/CreateTableDesc.java | 2 +-
.../ql/ddl/table/CreateTableLikeOperation.java | 14 +-
.../queries/clientpositive/partition_discovery.q | 16 ++
.../test/results/clientpositive/create_like.q.out | 1 -
.../results/clientpositive/create_like_view.q.out | 1 -
.../clientpositive/default_file_format.q.out | 4 -
.../results/clientpositive/druid/druid_topn.q.out | 1 -
.../druid/druidkafkamini_basic.q.out | 2 -
.../druid/druidmini_expressions.q.out | 2 -
.../test/results/clientpositive/druid_topn.q.out | 1 -
.../kafka/kafka_storage_handler.q.out | 4 -
.../clientpositive/llap/external_table_purge.q.out | 2 -
.../test/results/clientpositive/llap/mm_exim.q.out | 1 -
.../llap/strict_managed_tables2.q.out | 2 -
.../clientpositive/llap/whroot_external1.q.out | 6 -
.../clientpositive/partition_discovery.q.out | 197 +
.../clientpositive/show_create_table_alter.q.out | 5 -
.../clientpositive/show_create_table_serde.q.out | 1 -
.../clientpositive/spark/stats_noscan_2.q.out | 2 -
.../results/clientpositive/stats_noscan_2.q.out| 2 -
.../temp_table_display_colstats_tbllvl.q.out | 5 -
.../org/apache/hadoop/hive/metastore/Msck.java | 9 +-
27 files changed, 229 insertions(+), 62 deletions(-)
diff --git a/hbase-handler/src/test/results/positive/external_table_ppd.q.out
b/hbase-handler/src/test/results/positive/external_table_ppd.q.out
index 40abb42..139e35b 100644
--- a/hbase-handler/src/test/results/positive/external_table_ppd.q.out
+++ b/hbase-handler/src/test/results/positive/external_table_ppd.q.out
@@ -60,7 +60,6 @@ Table Parameters:
COLUMN_STATS_ACCURATE
{\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"bigint_col\":\"true\",\"boolean_col\":\"true\",\"double_col\":\"true\",\"float_col\":\"true\",\"int_col\":\"true\",\"key\":\"true\",\"smallint_col\":\"true\",\"tinyint_col\":\"true\"}}
EXTERNALTRUE
bucketing_version 2
- discover.partitions true
external.table.purgetrue
hbase.table.default.storage.typebinary
hbase.table.namet_hive
diff --git
a/hbase-handler/src/test/results/positive/hbase_binary_storage_queries.q.out
b/hbase-handler/src/test/results/positive/hbase_binary_storage_queries.q.out
index bf1a89d..1209c88 100644
--- a/hbase-handler/src/test/results/positive/hbase_binary_storage_queries.q.out
+++ b/hbase-handler/src/test/results/positive/hbase_binary_storage_queries.q.out
@@ -60,7 +60,6 @@ Table Parameters:
COLUMN_STATS_ACCURATE
{\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"bigint_col\":\"true\",\"boolean_col\":\"true\",\"double_col\":\"true\",\"float_col\":\"true\",\"int_col\":\"true\",\"key\":\"true\",\"smallint_col\":\"true\",\"tinyint_col\":\"true\"}}
EXTERNALTRUE
bucketing_version 2
- discover.partitions true
external.table.purgetrue
hbase.table.default.storage.typebinary
hbase.table.namet_hive
@@ -243,7 +242,6 @@ Table Parameters:
COLUMN_STATS_ACCURATE
{\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"bigint_col\":\"true\",\"boolean_col\":\"true\",\"double_col\":\"true\",\"float_col\":\"true\",\"int_col\":\"true\",\"key\":\"true\",\"smallint_col\":\"true\",\"tinyint_col\":\"true\"}}
EXTERNALTRUE
bucketing_version 2
- discover.partitions true
hbase.table.namet_hive
numFiles0
numRows 0
diff --git
[hive] branch master updated: HIVE-21446 : Hive Server going OOM during hive external table replications. (Mahesh Kumar Behera, reviewed by Sankar Hariappan)
This is an automated email from the ASF dual-hosted git repository.
mahesh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 38682a4 HIVE-21446 : Hive Server going OOM during hive external table
replications. (Mahesh Kumar Behera, reviewed by Sankar Hariappan)
38682a4 is described below
commit 38682a414708d810937012bae4b0c97deca5ef07
Author: Mahesh Kumar Behera
AuthorDate: Thu Mar 21 20:27:38 2019 +0530
HIVE-21446 : Hive Server going OOM during hive external table replications.
(Mahesh Kumar Behera, reviewed by Sankar Hariappan)
---
.../org/apache/hadoop/hive/common/FileUtils.java | 10 +-
.../apache/hadoop/hive/common/TestFileUtils.java | 11 +-
.../apache/hadoop/hive/ql/exec/ReplCopyTask.java | 2 +-
.../ql/exec/repl/ExternalTableCopyTaskBuilder.java | 86
.../hadoop/hive/ql/parse/repl/CopyUtils.java | 150 ++---
.../hive/ql/parse/repl/dump/io/FileOperations.java | 5 +-
.../hadoop/hive/ql/parse/repl/TestCopyUtils.java | 8 +-
.../apache/hadoop/hive/shims/Hadoop23Shims.java| 5 +-
.../org/apache/hadoop/hive/shims/HadoopShims.java | 5 +-
9 files changed, 182 insertions(+), 100 deletions(-)
diff --git a/common/src/java/org/apache/hadoop/hive/common/FileUtils.java
b/common/src/java/org/apache/hadoop/hive/common/FileUtils.java
index 23a3a6b..8b03faa 100644
--- a/common/src/java/org/apache/hadoop/hive/common/FileUtils.java
+++ b/common/src/java/org/apache/hadoop/hive/common/FileUtils.java
@@ -636,18 +636,18 @@ public final class FileUtils {
}
public static boolean distCp(FileSystem srcFS, List srcPaths, Path dst,
- boolean deleteSource, String doAsUser,
+ boolean deleteSource, UserGroupInformation proxyUser,
HiveConf conf, HadoopShims shims) throws IOException {
LOG.debug("copying srcPaths : {}, to DestPath :{} ,with doAs: {}",
-StringUtils.join(",", srcPaths), dst.toString(), doAsUser);
+StringUtils.join(",", srcPaths), dst.toString(), proxyUser);
boolean copied = false;
-if (doAsUser == null){
+if (proxyUser == null){
copied = shims.runDistCp(srcPaths, dst, conf);
} else {
- copied = shims.runDistCpAs(srcPaths, dst, conf, doAsUser);
+ copied = shims.runDistCpAs(srcPaths, dst, conf, proxyUser);
}
if (copied && deleteSource) {
- if (doAsUser != null) {
+ if (proxyUser != null) {
// if distcp is done using doAsUser, delete also should be done using
same user.
//TODO : Need to change the delete execution within doAs if doAsUser
is given.
throw new IOException("Distcp is called with doAsUser and delete
source set as true");
diff --git a/common/src/test/org/apache/hadoop/hive/common/TestFileUtils.java
b/common/src/test/org/apache/hadoop/hive/common/TestFileUtils.java
index b45832e..9b5748e 100644
--- a/common/src/test/org/apache/hadoop/hive/common/TestFileUtils.java
+++ b/common/src/test/org/apache/hadoop/hive/common/TestFileUtils.java
@@ -42,6 +42,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.shims.HadoopShims;
+import org.apache.hadoop.security.UserGroupInformation;
import org.junit.Assert;
import org.junit.Test;
@@ -239,14 +240,16 @@ public class TestFileUtils {
FileSystem fs = copySrc.getFileSystem(conf);
String doAsUser = conf.getVar(HiveConf.ConfVars.HIVE_DISTCP_DOAS_USER);
+UserGroupInformation proxyUser = UserGroupInformation.createProxyUser(
+doAsUser, UserGroupInformation.getLoginUser());
HadoopShims shims = mock(HadoopShims.class);
-when(shims.runDistCpAs(Collections.singletonList(copySrc), copyDst, conf,
doAsUser)).thenReturn(true);
+when(shims.runDistCpAs(Collections.singletonList(copySrc), copyDst, conf,
proxyUser)).thenReturn(true);
when(shims.runDistCp(Collections.singletonList(copySrc), copyDst,
conf)).thenReturn(false);
// doAs when asked
-Assert.assertTrue(FileUtils.distCp(fs, Collections.singletonList(copySrc),
copyDst, false, doAsUser, conf, shims));
-verify(shims).runDistCpAs(Collections.singletonList(copySrc), copyDst,
conf, doAsUser);
+Assert.assertTrue(FileUtils.distCp(fs, Collections.singletonList(copySrc),
copyDst, false, proxyUser, conf, shims));
+verify(shims).runDistCpAs(Collections.singletonList(copySrc), copyDst,
conf, proxyUser);
// don't doAs when not asked
Assert.assertFalse(FileUtils.distCp(fs,
Collections.singletonList(copySrc), copyDst, true, null, conf, shims));
verify(shims).runDistCp(Collections.singletonList(copySrc), copyDst, conf);
@@ -254,7 +257,7 @@ public class TestFileUtils {
// When distcp is done with doAs, the delete should also be done as doAs.
But in current code its broken. This
// should be fixed. For now check is added to avoid wrong usage. So if
doAs is
