[hive] branch master updated: HIVE-22114 : insert query for partitioned insert only table failing when all buckets are empty (Vineet Garg via Ashutosh Chauhan)
This is an automated email from the ASF dual-hosted git repository.
hashutosh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new a501e6e HIVE-22114 : insert query for partitioned insert only table
failing when all buckets are empty (Vineet Garg via Ashutosh Chauhan)
a501e6e is described below
commit a501e6e96c9a27dd2638176d7137eece105bc4cd
Author: Vineet Garg
AuthorDate: Wed Aug 14 20:18:29 2019 -0700
HIVE-22114 : insert query for partitioned insert only table failing when
all buckets are empty (Vineet Garg via Ashutosh Chauhan)
Signed-off-by: Ashutosh Chauhan
---
.../test/resources/testconfiguration.properties| 1 +
.../org/apache/hadoop/hive/ql/exec/Utilities.java | 4 +-
.../clientpositive/insert_only_empty_query.q | 24 +++
.../llap/insert_only_empty_query.q.out | 222 +
4 files changed, 249 insertions(+), 2 deletions(-)
diff --git a/itests/src/test/resources/testconfiguration.properties
b/itests/src/test/resources/testconfiguration.properties
index 8061c1a..b8e97bd 100644
--- a/itests/src/test/resources/testconfiguration.properties
+++ b/itests/src/test/resources/testconfiguration.properties
@@ -553,6 +553,7 @@ minillaplocal.query.files=\
insert_dir_distcp.q,\
insert_into_default_keyword.q,\
insert_into_with_schema.q,\
+ insert_only_empty_query.q, \
insert_overwrite.q,\
insert_values_orig_table.q,\
insert_values_orig_table_use_metadata.q,\
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
index ac89dd9..d59ca8c 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
@@ -4231,8 +4231,8 @@ public final class Utilities {
Utilities.FILE_OP_LOGGER.debug("Looking for files in: {}", specPath);
AcidUtils.IdPathFilter filter = new AcidUtils.IdPathFilter(writeId,
stmtId);
-if (isMmCtas && !fs.exists(specPath)) {
- Utilities.FILE_OP_LOGGER.info("Creating table directory for CTAS with no
output at {}", specPath);
+if (!fs.exists(specPath)) {
+ Utilities.FILE_OP_LOGGER.info("Creating directory with no output at {}",
specPath);
FileUtils.mkdir(fs, specPath, hconf);
}
Path[] files = getMmDirectoryCandidates(
diff --git a/ql/src/test/queries/clientpositive/insert_only_empty_query.q
b/ql/src/test/queries/clientpositive/insert_only_empty_query.q
new file mode 100644
index 000..5e731eb
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/insert_only_empty_query.q
@@ -0,0 +1,24 @@
+set hive.explain.user=false;
+set hive.fetch.task.conversion=none;
+set hive.exec.dynamic.partition.mode=nonstrict;
+set hive.support.concurrency=true;
+set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager;
+
+create table src_emptybucket_partitioned_1 (name string, age int, gpa
decimal(3,2))
+ partitioned by(year int)
+ clustered by (age)
+ sorted by (age)
+ into 100 buckets
+ stored as orc tblproperties
("transactional"="true", "transactional_properties"="insert_only");
+
+create table source_table(name string, age int, gpa decimal(3,2));
+insert into source_table values("name", 56, 4);
+
+explain insert into table src_emptybucket_partitioned_1 partition(year=2015)
select * from source_table limit 0;
+insert into table src_emptybucket_partitioned_1 partition(year=2015) select *
from source_table limit 0;
+
+insert into table src_emptybucket_partitioned_1 partition(year=2015) select *
from source_table limit 1;
+select * from src_emptybucket_partitioned_1;
+
+drop table src_emptybucket_partitioned_1;
+drop table source_table;
diff --git
a/ql/src/test/results/clientpositive/llap/insert_only_empty_query.q.out
b/ql/src/test/results/clientpositive/llap/insert_only_empty_query.q.out
new file mode 100644
index 000..bfa72c5
--- /dev/null
+++ b/ql/src/test/results/clientpositive/llap/insert_only_empty_query.q.out
@@ -0,0 +1,222 @@
+PREHOOK: query: create table src_emptybucket_partitioned_1 (name string, age
int, gpa decimal(3,2))
+ partitioned by(year int)
+ clustered by (age)
+ sorted by (age)
+ into 100 buckets
+ stored as orc tblproperties
("transactional"="true", "transactional_properties"="insert_only")
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@src_emptybucket_partitioned_1
+POSTHOOK: query: create table src_emptybucket_partitioned_1 (name string, age
int, gpa decimal(3,2))
+ partitioned by(year int)
+
[hive] branch branch-3 updated: HIVE-20057 For ALTER TABLE t SET TBLPROPERTIES ('EXTERNAL'='TRUE'); `TBL_TYPE` attribute change not reflecting for non-CAPS (Anirudh via Alan Gates)
This is an automated email from the ASF dual-hosted git repository.
gates pushed a commit to branch branch-3
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/branch-3 by this push:
new aa9d880 HIVE-20057 For ALTER TABLE t SET TBLPROPERTIES
('EXTERNAL'='TRUE'); `TBL_TYPE` attribute change not reflecting for non-CAPS
(Anirudh via Alan Gates)
aa9d880 is described below
commit aa9d8809ea8ebedc297e4fe8ef1b157bb011aff6
Author: Alan Gates
AuthorDate: Wed Aug 14 15:40:09 2019 -0700
HIVE-20057 For ALTER TABLE t SET TBLPROPERTIES ('EXTERNAL'='TRUE');
`TBL_TYPE` attribute change not reflecting for non-CAPS (Anirudh via Alan Gates)
---
.../src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git
a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
index 1a4f98b..8f1d3e0 100644
---
a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
+++
b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
@@ -1793,12 +1793,12 @@ public class ObjectStore implements RawStore,
Configurable {
// accordingly
String tableType = tbl.getTableType();
boolean isExternal =
Boolean.parseBoolean(tbl.getParameters().get("EXTERNAL"));
-if (TableType.MANAGED_TABLE.toString().equals(tableType)) {
+if (TableType.MANAGED_TABLE.toString().equalsIgnoreCase(tableType)) {
if (isExternal) {
tableType = TableType.EXTERNAL_TABLE.toString();
}
}
-if (TableType.EXTERNAL_TABLE.toString().equals(tableType)) {
+if (TableType.EXTERNAL_TABLE.toString().equalsIgnoreCase(tableType)) {
if (!isExternal) {
tableType = TableType.MANAGED_TABLE.toString();
}
[hive] branch master updated: HIVE-22109: Hive.renamePartition expects catalog name to be set (Naveen Gangam, reviewed by Thejas Nair)
This is an automated email from the ASF dual-hosted git repository.
ngangam pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new f9bd589 HIVE-22109: Hive.renamePartition expects catalog name to be
set (Naveen Gangam, reviewed by Thejas Nair)
f9bd589 is described below
commit f9bd589f601e3b0bce668de0d737979284bd6150
Author: Naveen Gangam
AuthorDate: Wed Aug 14 21:02:36 2019 -0400
HIVE-22109: Hive.renamePartition expects catalog name to be set (Naveen
Gangam, reviewed by Thejas Nair)
---
ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
index 21b1ef6..a958657 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
@@ -956,7 +956,8 @@ public class Hive {
}
}
- getMSC().renamePartition(tbl.getCatName(), tbl.getDbName(),
tbl.getTableName(), pvals,
+ String catName = (tbl.getCatalogName() != null) ? tbl.getCatalogName() :
getDefaultCatalog(conf);
+ getMSC().renamePartition(catName, tbl.getDbName(), tbl.getTableName(),
pvals,
newPart.getTPartition(), validWriteIds);
} catch (InvalidOperationException e){
[hive] branch master updated: HIVE-22112 : update jackson version in disconnected poms (Ashutosh Chauhan via Vineet Garg)
This is an automated email from the ASF dual-hosted git repository.
hashutosh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new ff463aa HIVE-22112 : update jackson version in disconnected poms
(Ashutosh Chauhan via Vineet Garg)
ff463aa is described below
commit ff463aa34483915db73e2e38e735f4e49ae2927d
Author: Ashutosh Chauhan
AuthorDate: Wed Aug 14 10:32:39 2019 -0700
HIVE-22112 : update jackson version in disconnected poms (Ashutosh Chauhan
via Vineet Garg)
---
standalone-metastore/pom.xml | 2 +-
testutils/ptest2/pom.xml | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/standalone-metastore/pom.xml b/standalone-metastore/pom.xml
index be1f452..2655545 100644
--- a/standalone-metastore/pom.xml
+++ b/standalone-metastore/pom.xml
@@ -82,7 +82,7 @@
19.0
3.1.0
2.6.1
-2.9.8
+2.9.9
5.5.1
4.11
0.9.3
diff --git a/testutils/ptest2/pom.xml b/testutils/ptest2/pom.xml
index c5f6e1a..e8f964c 100644
--- a/testutils/ptest2/pom.xml
+++ b/testutils/ptest2/pom.xml
@@ -31,7 +31,7 @@ limitations under the License.
2.1.0
${basedir}/../../checkstyle/
2.12.1
-2.9.8
+2.9.9
[hive] branch branch-2 updated: HIVE-22096 Backport HIVE-21584 to branch-2 (Yuming Wang via Alan Gates).
This is an automated email from the ASF dual-hosted git repository.
gates pushed a commit to branch branch-2
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/branch-2 by this push:
new 7d5a3d6 HIVE-22096 Backport HIVE-21584 to branch-2 (Yuming Wang via
Alan Gates).
7d5a3d6 is described below
commit 7d5a3d6e58b6c10fe4657eb9861f74d02d45b270
Author: Alan Gates
AuthorDate: Tue Aug 13 15:17:22 2019 -0700
HIVE-22096 Backport HIVE-21584 to branch-2 (Yuming Wang via Alan Gates).
---
.../src/java/org/apache/hive/beeline/Commands.java | 2 +-
.../hive/beeline/TestClassNameCompleter.java | 4 +-
.../org/apache/hadoop/hive/common/JavaUtils.java | 57 ++--
.../hive/llap/daemon/impl/FunctionLocalizer.java | 18 ++-
.../hadoop/hive/metastore/MetaStoreUtils.java | 21 ++-
.../hadoop/hive/ql/exec/AddToClassPathAction.java | 92 +
.../org/apache/hadoop/hive/ql/exec/Utilities.java | 100 +++---
.../apache/hadoop/hive/ql/exec/mr/ExecDriver.java | 7 +-
.../apache/hadoop/hive/ql/exec/mr/ExecMapper.java | 14 +-
.../apache/hadoop/hive/ql/exec/mr/ExecReducer.java | 16 +--
.../hive/ql/exec/spark/SparkRecordHandler.java | 14 +-
.../hadoop/hive/ql/exec/tez/RecordProcessor.java | 17 +--
.../hadoop/hive/ql/session/SessionState.java | 32 +++--
.../hive/ql/exec/TestAddToClassPathAction.java | 145 +
.../hive/spark/client/SparkClientUtilities.java| 23 +++-
15 files changed, 371 insertions(+), 191 deletions(-)
diff --git a/beeline/src/java/org/apache/hive/beeline/Commands.java
b/beeline/src/java/org/apache/hive/beeline/Commands.java
index 35b3b3b..fb5d8f5 100644
--- a/beeline/src/java/org/apache/hive/beeline/Commands.java
+++ b/beeline/src/java/org/apache/hive/beeline/Commands.java
@@ -169,7 +169,7 @@ public class Commands {
return false;
}
-URLClassLoader classLoader = (URLClassLoader)
Thread.currentThread().getContextClassLoader();
+ClassLoader classLoader = Thread.currentThread().getContextClassLoader();
try {
beeLine.debug(jarPath + " is added to the local beeline.");
URLClassLoader newClassLoader = new URLClassLoader(new URL[]{p.toURL()},
classLoader);
diff --git
a/beeline/src/test/org/apache/hive/beeline/TestClassNameCompleter.java
b/beeline/src/test/org/apache/hive/beeline/TestClassNameCompleter.java
index 137..6c3e57f 100644
--- a/beeline/src/test/org/apache/hive/beeline/TestClassNameCompleter.java
+++ b/beeline/src/test/org/apache/hive/beeline/TestClassNameCompleter.java
@@ -40,7 +40,7 @@ public class TestClassNameCompleter {
String fileName = "empty.file.jar";
File p = tmpFolder.newFile(fileName);
-URLClassLoader classLoader = (URLClassLoader)
Thread.currentThread().getContextClassLoader();
+ClassLoader classLoader = Thread.currentThread().getContextClassLoader();
try {
URLClassLoader newClassLoader = new URLClassLoader(new URL[] { p.toURL()
}, classLoader);
@@ -62,7 +62,7 @@ public class TestClassNameCompleter {
String fileName = "empty.file";
File p = tmpFolder.newFile(fileName);
-URLClassLoader classLoader = (URLClassLoader)
Thread.currentThread().getContextClassLoader();
+ClassLoader classLoader = Thread.currentThread().getContextClassLoader();
try {
URLClassLoader newClassLoader = new URLClassLoader(new URL[] { p.toURL()
}, classLoader);
diff --git a/common/src/java/org/apache/hadoop/hive/common/JavaUtils.java
b/common/src/java/org/apache/hadoop/hive/common/JavaUtils.java
index 3916fe3..c53d1a2 100644
--- a/common/src/java/org/apache/hadoop/hive/common/JavaUtils.java
+++ b/common/src/java/org/apache/hadoop/hive/common/JavaUtils.java
@@ -18,12 +18,8 @@
package org.apache.hadoop.hive.common;
-import java.io.ByteArrayOutputStream;
import java.io.Closeable;
import java.io.IOException;
-import java.io.PrintStream;
-import java.lang.reflect.InvocationTargetException;
-import java.lang.reflect.Method;
import java.net.URLClassLoader;
import java.util.Arrays;
import java.util.List;
@@ -38,22 +34,6 @@ import org.slf4j.LoggerFactory;
public final class JavaUtils {
private static final Logger LOG = LoggerFactory.getLogger(JavaUtils.class);
- private static final Method SUN_MISC_UTIL_RELEASE;
-
- static {
-if (Closeable.class.isAssignableFrom(URLClassLoader.class)) {
- SUN_MISC_UTIL_RELEASE = null;
-} else {
- Method release = null;
- try {
-Class clazz = Class.forName("sun.misc.ClassLoaderUtil");
-release = clazz.getMethod("releaseLoader", URLClassLoader.class);
- } catch (Exception e) {
-// ignore
- }
- SUN_MISC_UTIL_RELEASE = release;
-}
- }
/**
* Standard way of getting classloader in Hive code (outside of Hadoop).
@@ -87,8 +67,10 @@ public final class JavaUtils {
try {
closeClassLoader(current);
} catch (IOException
[hive] branch master updated: HIVE-13457 : Create HS2 REST API endpoints for monitoring information (Pawel Szostek via Szehon)
This is an automated email from the ASF dual-hosted git repository.
szehon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 71605e6 HIVE-13457 : Create HS2 REST API endpoints for monitoring
information (Pawel Szostek via Szehon)
71605e6 is described below
commit 71605e65ef0fa6a98036bb956db80b285835e1f3
Author: Szehon Ho
AuthorDate: Wed Aug 7 10:57:38 2019 -0700
HIVE-13457 : Create HS2 REST API endpoints for monitoring information
(Pawel Szostek via Szehon)
---
service/pom.xml| 12 ++
.../apache/hive/service/server/HiveServer2.java| 7 +
.../service/servlet/QueriesRESTfulAPIServlet.java | 178
.../hive/service/server/TestHS2HttpServer.java | 181 ++---
4 files changed, 352 insertions(+), 26 deletions(-)
diff --git a/service/pom.xml b/service/pom.xml
index c73a621..a75021e 100644
--- a/service/pom.xml
+++ b/service/pom.xml
@@ -300,6 +300,18 @@
${apache-directory-server.version}
test
+
+ org.powermock
+ powermock-module-junit4
+ ${powermock.version}
+ test
+
+
+ org.powermock
+ powermock-api-mockito
+ ${powermock.version}
+ test
+
diff --git a/service/src/java/org/apache/hive/service/server/HiveServer2.java
b/service/src/java/org/apache/hive/service/server/HiveServer2.java
index 2f3767f..5d81668 100644
--- a/service/src/java/org/apache/hive/service/server/HiveServer2.java
+++ b/service/src/java/org/apache/hive/service/server/HiveServer2.java
@@ -106,6 +106,7 @@ import org.apache.hive.service.cli.thrift.ThriftCLIService;
import org.apache.hive.service.cli.thrift.ThriftHttpCLIService;
import org.apache.hive.service.servlet.HS2LeadershipStatus;
import org.apache.hive.service.servlet.HS2Peers;
+import org.apache.hive.service.servlet.QueriesRESTfulAPIServlet;
import org.apache.hive.service.servlet.QueryProfileServlet;
import org.apache.http.StatusLine;
import org.apache.http.client.methods.CloseableHttpResponse;
@@ -171,6 +172,11 @@ public class HiveServer2 extends CompositeService {
}
@VisibleForTesting
+ public CLIService getCliService() {
+return this.cliService;
+ }
+
+ @VisibleForTesting
public void setPamAuthenticator(PamAuthenticator pamAuthenticator) {
this.pamAuthenticator = pamAuthenticator;
}
@@ -403,6 +409,7 @@ public class HiveServer2 extends CompositeService {
webServer = builder.build();
webServer.addServlet("query_page", "/query_page",
QueryProfileServlet.class);
+ webServer.addServlet("api", "/api/*",
QueriesRESTfulAPIServlet.class);
}
}
} catch (IOException ie) {
diff --git
a/service/src/java/org/apache/hive/service/servlet/QueriesRESTfulAPIServlet.java
b/service/src/java/org/apache/hive/service/servlet/QueriesRESTfulAPIServlet.java
new file mode 100644
index 000..990ec6e
--- /dev/null
+++
b/service/src/java/org/apache/hive/service/servlet/QueriesRESTfulAPIServlet.java
@@ -0,0 +1,178 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hive.service.servlet;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hive.ql.QueryInfo;
+import org.apache.hive.service.cli.operation.OperationManager;
+import org.apache.hive.service.cli.session.HiveSession;
+import org.apache.hive.service.cli.session.SessionManager;
+import org.codehaus.jackson.JsonGenerator;
+import org.codehaus.jackson.JsonProcessingException;
+import org.codehaus.jackson.Version;
+import org.codehaus.jackson.map.JsonSerializer;
+import org.codehaus.jackson.map.ObjectMapper;
+import org.codehaus.jackson.map.SerializerProvider;
+import org.codehaus.jackson.map.module.SimpleModule;
+
+import javax.servlet.ServletContext;
+import javax.servlet.ServletException;
+import javax.servlet.http.HttpServlet;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.util.Collection;
+
+/**
+ *
