[hive] branch master updated: HIVE-22476: Hive datediff function provided inconsistent results when hive.fetch.task.conversion is set to none (Slim Bouguerra via Jesus Camacho Rodriguez)
This is an automated email from the ASF dual-hosted git repository.
jcamacho pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new b45e152 HIVE-22476: Hive datediff function provided inconsistent
results when hive.fetch.task.conversion is set to none (Slim Bouguerra via
Jesus Camacho Rodriguez)
b45e152 is described below
commit b45e152745cbe8f2ae8c0651c84ae3ce0552e2ff
Author: Slim Bouguerra
AuthorDate: Tue Dec 3 08:57:26 2019 -0800
HIVE-22476: Hive datediff function provided inconsistent results when
hive.fetch.task.conversion is set to none (Slim Bouguerra via Jesus Camacho
Rodriguez)
Close apache/hive#1046
---
.../org/apache/hadoop/hive/common/type/Date.java | 5 +
.../hadoop/hive/common/type/TimestampTZUtil.java | 1 -
.../ql/udf/generic/TestGenericUDFDateDiff.java | 24 +-
.../hive/ql/udf/generic/TestGenericUDFLastDay.java | 2 +-
.../hive/ql/udf/generic/TestGenericUDFNextDay.java | 2 +-
ql/src/test/queries/clientpositive/date_udf.q | 6 +
.../results/clientpositive/llap/date_udf.q.out | 45
.../results/clientpositive/spark/date_udf.q.out| 254 -
8 files changed, 78 insertions(+), 261 deletions(-)
diff --git a/common/src/java/org/apache/hadoop/hive/common/type/Date.java
b/common/src/java/org/apache/hadoop/hive/common/type/Date.java
index 6ecfcf6..43f5977 100644
--- a/common/src/java/org/apache/hadoop/hive/common/type/Date.java
+++ b/common/src/java/org/apache/hadoop/hive/common/type/Date.java
@@ -127,6 +127,11 @@ public class Date implements Comparable {
int idx = s.indexOf(" ");
if (idx != -1) {
s = s.substring(0, idx);
+} else {
+ idx = s.indexOf('T');
+ if (idx != -1) {
+s = s.substring(0, idx);
+ }
}
LocalDate localDate;
try {
diff --git
a/common/src/java/org/apache/hadoop/hive/common/type/TimestampTZUtil.java
b/common/src/java/org/apache/hadoop/hive/common/type/TimestampTZUtil.java
index 4708d35..862acb8 100644
--- a/common/src/java/org/apache/hadoop/hive/common/type/TimestampTZUtil.java
+++ b/common/src/java/org/apache/hadoop/hive/common/type/TimestampTZUtil.java
@@ -53,7 +53,6 @@ public class TimestampTZUtil {
builder.optionalStart().appendLiteral("
").append(DateTimeFormatter.ofPattern("HH:mm:ss")).
optionalStart().appendFraction(ChronoField.NANO_OF_SECOND, 1, 9, true).
optionalEnd().optionalEnd();
-
// Zone part
builder.optionalStart().appendLiteral(" ").optionalEnd();
builder.optionalStart().appendZoneText(TextStyle.NARROW).optionalEnd();
diff --git
a/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFDateDiff.java
b/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFDateDiff.java
index 86b914d..6151bab 100644
---
a/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFDateDiff.java
+++
b/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFDateDiff.java
@@ -18,6 +18,10 @@
package org.apache.hadoop.hive.ql.udf.generic;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+
+import java.text.ParseException;
import java.time.LocalDateTime;
import org.apache.hadoop.hive.common.type.Date;
@@ -32,15 +36,27 @@ import
org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectIn
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
-
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
import org.junit.Test;
-/**
- * TestGenericUDFDateDiff.
- */
public class TestGenericUDFDateDiff {
+
+ @Test
+ public void testStringToDateISOFormat() throws HiveException, ParseException
{
+GenericUDFDateDiff udf = new GenericUDFDateDiff();
+ObjectInspector valueOI1 =
PrimitiveObjectInspectorFactory.javaStringObjectInspector;
+ObjectInspector valueOI2 =
PrimitiveObjectInspectorFactory.javaStringObjectInspector;
+ObjectInspector[] arguments = {valueOI1, valueOI2};
+
+udf.initialize(arguments);
+DeferredObject valueObj1 = new DeferredJavaObject(new
Text("2019-09-09T10:45:49+02:00"));
+DeferredObject valueObj2 = new DeferredJavaObject(new Text("2019-11-07
23:20:39.503"));
+DeferredObject[] args = {valueObj1, valueObj2};
+IntWritable output = udf.evaluate(args);
+assertEquals("date_iff() test for STRING failed ", "-59",
output.toString());
+ }
+
@Test
public void testStringToDate() throws HiveException {
GenericUDFDateDiff udf = new GenericUDFDateDiff();
diff --git
a/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFLastDay.java
b/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFLastDay.java
index 3cbcbdb..6f4da18 100644
---
a/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFLastDay.java
+++
[hive] branch master updated: HIVE-23498: Disable HTTP Trace method on ThriftHttpCliService (Rajkumar Singh, reviewed by Naveen Gangam)
This is an automated email from the ASF dual-hosted git repository.
rajksingh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 4cc3ae9 HIVE-23498: Disable HTTP Trace method on ThriftHttpCliService
(Rajkumar Singh, reviewed by Naveen Gangam)
4cc3ae9 is described below
commit 4cc3ae97d48b359a47c3608c7b307c58233e088d
Author: Rajkumar Singh
AuthorDate: Mon Jun 1 20:05:12 2020 -0700
HIVE-23498: Disable HTTP Trace method on ThriftHttpCliService (Rajkumar
Singh, reviewed by Naveen Gangam)
---
.../service/cli/thrift/ThriftHttpCLIService.java | 26 ++
1 file changed, 26 insertions(+)
diff --git
a/service/src/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java
b/service/src/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java
index 6652668..8524c12 100644
---
a/service/src/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java
+++
b/service/src/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java
@@ -45,6 +45,8 @@ import org.apache.thrift.protocol.TProtocolFactory;
import org.apache.thrift.server.TServlet;
import org.eclipse.jetty.io.Connection;
import org.eclipse.jetty.io.EndPoint;
+import org.eclipse.jetty.security.ConstraintMapping;
+import org.eclipse.jetty.security.ConstraintSecurityHandler;
import org.eclipse.jetty.server.Connector;
import org.eclipse.jetty.server.HttpConfiguration;
import org.eclipse.jetty.server.HttpConnectionFactory;
@@ -53,6 +55,7 @@ import org.eclipse.jetty.server.ServerConnector;
import org.eclipse.jetty.server.handler.gzip.GzipHandler;
import org.eclipse.jetty.servlet.ServletContextHandler;
import org.eclipse.jetty.servlet.ServletHolder;
+import org.eclipse.jetty.util.security.Constraint;
import org.eclipse.jetty.util.ssl.SslContextFactory;
import org.eclipse.jetty.util.thread.ExecutorThreadPool;
@@ -192,6 +195,7 @@ public class ThriftHttpCLIService extends ThriftCLIService {
server.setHandler(context);
}
context.addServlet(new ServletHolder(thriftHttpServlet), httpPath);
+ constrainHttpMethods(context, false);
// TODO: check defaults: maxTimeout, keepalive, maxBodySize,
// bodyRecieveDuration, etc.
@@ -269,6 +273,28 @@ public class ThriftHttpCLIService extends ThriftCLIService
{
return httpPath;
}
+ public void constrainHttpMethods(ServletContextHandler ctxHandler, boolean
allowOptionsMethod) {
+Constraint c = new Constraint();
+c.setAuthenticate(true);
+
+ConstraintMapping cmt = new ConstraintMapping();
+cmt.setConstraint(c);
+cmt.setMethod("TRACE");
+cmt.setPathSpec("/*");
+
+ConstraintSecurityHandler securityHandler = new
ConstraintSecurityHandler();
+if (!allowOptionsMethod) {
+ ConstraintMapping cmo = new ConstraintMapping();
+ cmo.setConstraint(c);
+ cmo.setMethod("OPTIONS");
+ cmo.setPathSpec("/*");
+ securityHandler.setConstraintMappings(new ConstraintMapping[] {cmt,
cmo});
+} else {
+ securityHandler.setConstraintMappings(new ConstraintMapping[] {cmt});
+}
+ctxHandler.setSecurityHandler(securityHandler);
+ }
+
@Override
protected void stopServer() {
if((server != null) && server.isStarted()) {
[hive] branch master updated: HIVE-21624: LLAP: Cpu metrics at thread level is broken (Prasanth Jayachandran reviewed by Ashutosh Chauhan)
This is an automated email from the ASF dual-hosted git repository.
prasanthj pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new e5ce723 HIVE-21624: LLAP: Cpu metrics at thread level is broken
(Prasanth Jayachandran reviewed by Ashutosh Chauhan)
e5ce723 is described below
commit e5ce7232d6182e20d87a743c6f73fdfb566119f5
Author: Prasanth Jayachandran
AuthorDate: Mon Jun 1 17:28:34 2020 -0700
HIVE-21624: LLAP: Cpu metrics at thread level is broken (Prasanth
Jayachandran reviewed by Ashutosh Chauhan)
---
.../org/apache/hadoop/hive/llap/daemon/impl/TaskExecutorService.java | 3 ++-
.../org/apache/hadoop/hive/llap/metrics/LlapDaemonExecutorMetrics.java | 3 ++-
2 files changed, 4 insertions(+), 2 deletions(-)
diff --git
a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/TaskExecutorService.java
b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/TaskExecutorService.java
index 1d6e852..3195ea7 100644
---
a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/TaskExecutorService.java
+++
b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/TaskExecutorService.java
@@ -91,7 +91,8 @@ import static
com.google.common.base.Preconditions.checkNotNull;
public class TaskExecutorService extends AbstractService
implements Scheduler,
SchedulerFragmentCompletingListener {
private static final Logger LOG =
LoggerFactory.getLogger(TaskExecutorService.class);
- private static final String TASK_EXECUTOR_THREAD_NAME_FORMAT =
"Task-Executor-%d";
+ public static final String TASK_EXECUTOR_THREAD_NAME_FORMAT_PREFIX =
"Task-Executor-";
+ private static final String TASK_EXECUTOR_THREAD_NAME_FORMAT =
TASK_EXECUTOR_THREAD_NAME_FORMAT_PREFIX + "%d";
private static final String WAIT_QUEUE_SCHEDULER_THREAD_NAME_FORMAT =
"Wait-Queue-Scheduler-%d";
private static final long PREEMPTION_KILL_GRACE_MS = 500; // 500ms
private static final int PREEMPTION_KILL_GRACE_SLEEP_MS = 50; // 50ms
diff --git
a/llap-server/src/java/org/apache/hadoop/hive/llap/metrics/LlapDaemonExecutorMetrics.java
b/llap-server/src/java/org/apache/hadoop/hive/llap/metrics/LlapDaemonExecutorMetrics.java
index 57547c1..c1f91e1 100644
---
a/llap-server/src/java/org/apache/hadoop/hive/llap/metrics/LlapDaemonExecutorMetrics.java
+++
b/llap-server/src/java/org/apache/hadoop/hive/llap/metrics/LlapDaemonExecutorMetrics.java
@@ -71,6 +71,7 @@ import
org.apache.commons.math3.stat.descriptive.DescriptiveStatistics;
import
org.apache.commons.math3.stat.descriptive.SynchronizedDescriptiveStatistics;
import org.apache.hadoop.hive.common.JvmMetrics;
import org.apache.hadoop.hive.llap.daemon.impl.ContainerRunnerImpl;
+import org.apache.hadoop.hive.llap.daemon.impl.TaskExecutorService;
import org.apache.hadoop.metrics2.MetricsCollector;
import org.apache.hadoop.metrics2.MetricsInfo;
import org.apache.hadoop.metrics2.MetricsRecordBuilder;
@@ -213,7 +214,7 @@ public class LlapDaemonExecutorMetrics implements
MetricsSource {
this.userMetricsInfoMap.put(i, miu);
this.executorThreadCpuTime[i] = registry.newGauge(mic, 0L);
this.executorThreadUserTime[i] = registry.newGauge(miu, 0L);
- this.executorNames.put(ContainerRunnerImpl.THREAD_NAME_FORMAT_PREFIX +
i, i);
+
this.executorNames.put(TaskExecutorService.TASK_EXECUTOR_THREAD_NAME_FORMAT_PREFIX
+ i, i);
}
if (timedWindowAverageDataPoints > 0) {
[hive] branch master updated: HIVE-23435 : Full outer join result is missing rows (Mustafa Iman via Ashutosh Chauhan)
This is an automated email from the ASF dual-hosted git repository.
hashutosh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 779c42f HIVE-23435 : Full outer join result is missing rows (Mustafa
Iman via Ashutosh Chauhan)
779c42f is described below
commit 779c42fa3c73f41785eb3266f3438ab9e2a75032
Author: Mustafa Iman
AuthorDate: Mon Jun 1 10:26:14 2020 -0700
HIVE-23435 : Full outer join result is missing rows (Mustafa Iman via
Ashutosh Chauhan)
Signed-off-by: Ashutosh Chauhan
---
.../hadoop/hive/ql/exec/CommonJoinOperator.java| 18 +++-
.../hive/ql/exec/CommonMergeJoinOperator.java | 2 +-
.../apache/hadoop/hive/ql/exec/JoinOperator.java | 1 +
.../hadoop/hive/ql/exec/MapJoinOperator.java | 1 +
.../hadoop/hive/ql/exec/SMBMapJoinOperator.java| 1 +
.../ql/exec/vector/VectorMapJoinBaseOperator.java | 2 -
.../ql/exec/vector/VectorSMBMapJoinOperator.java | 1 -
.../clientpositive/vector_full_outer_join2.q | 28 +
.../results/clientpositive/llap/join_1to1.q.out| 84 ---
.../llap/vector_full_outer_join2.q.out | 113 +
10 files changed, 233 insertions(+), 18 deletions(-)
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonJoinOperator.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonJoinOperator.java
index 2d76848..7a7c8a5 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonJoinOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonJoinOperator.java
@@ -341,6 +341,8 @@ public abstract class CommonJoinOperator extends
forwardCache = new Object[totalSz];
aliasFilterTags = new short[numAliases];
Arrays.fill(aliasFilterTags, (byte)0xff);
+aliasFilterTagsNext = new short[numAliases];
+Arrays.fill(aliasFilterTagsNext, (byte) 0xff);
filterTags = new short[numAliases];
skipVectors = new boolean[numAliases][];
@@ -478,6 +480,7 @@ public abstract class CommonJoinOperator extends
* 100, 30 : N, N
*/
protected transient short[] aliasFilterTags;
+ protected transient short[] aliasFilterTagsNext;
// all evaluation should be processed here for valid aliasFilterTags
//
@@ -491,11 +494,21 @@ public abstract class CommonJoinOperator extends
short filterTag = JoinUtil.isFiltered(row, joinFilters[alias],
joinFilterObjectInspectors[alias], filterMaps[alias]);
nr.add(new ShortWritable(filterTag));
- aliasFilterTags[alias] &= filterTag;
}
return nr;
}
+ protected void addToAliasFilterTags(byte alias, List object, boolean
isNextGroup) {
+boolean hasFilter = hasFilter(alias);
+if (hasFilter) {
+ if (isNextGroup) {
+aliasFilterTagsNext[alias] &= ((ShortWritable)
(object.get(object.size() - 1))).get();
+ } else {
+aliasFilterTags[alias] &= ((ShortWritable) (object.get(object.size() -
1))).get();
+ }
+}
+ }
+
// fill forwardCache with skipvector
// returns whether a record was forwarded
private boolean createForwardJoinObject(boolean[] skip) throws HiveException
{
@@ -961,7 +974,8 @@ public abstract class CommonJoinOperator extends
genJoinObject();
}
}
-Arrays.fill(aliasFilterTags, (byte)0xff);
+System.arraycopy(aliasFilterTagsNext, 0, aliasFilterTags, 0,
aliasFilterTagsNext.length);
+Arrays.fill(aliasFilterTagsNext, (byte) 0xff);
}
protected void reportProgress() {
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonMergeJoinOperator.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonMergeJoinOperator.java
index 581577e..0d9dc46 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonMergeJoinOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonMergeJoinOperator.java
@@ -46,7 +46,6 @@ import org.apache.hadoop.hive.ql.plan.OperatorDesc;
import org.apache.hadoop.hive.ql.plan.api.OperatorType;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
import
org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils.ObjectInspectorCopyOption;
-import org.apache.hadoop.hive.serde2.objectinspector.StructField;
import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
import org.apache.hadoop.io.WritableComparable;
import org.apache.hadoop.io.WritableComparator;
@@ -242,6 +241,7 @@ public class CommonMergeJoinOperator extends
AbstractMapJoinOperator implements Serial
alias = (byte) tag;
List nr = getFilteredValue(alias, row);
+ addToAliasFilterTags(alias, nr, false);
if (handleSkewJoin) {
skewJoinKeyContext.handleSkew(tag);
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java
index 07b1fba..489d09f 100644
---
