[hive] branch master updated: HIVE-21426: Remove Utilities Global Random (David Mollitor, reviewed by Peter Vary)
This is an automated email from the ASF dual-hosted git repository.
dmollitor pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 40cd40d HIVE-21426: Remove Utilities Global Random (David Mollitor,
reviewed by Peter Vary)
40cd40d is described below
commit 40cd40d5e212860868d7dd15335c0085568416ce
Author: David Mollitor
AuthorDate: Mon Oct 21 18:46:42 2019 -0400
HIVE-21426: Remove Utilities Global Random (David Mollitor, reviewed by
Peter Vary)
---
.../hadoop/hive/ql/exec/SparkHashTableSinkOperator.java | 3 ++-
ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java | 15 +--
.../org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java | 10 +++---
.../org/apache/hadoop/hive/ql/io/merge/MergeFileTask.java | 4 +++-
.../hive/ql/io/rcfile/truncate/ColumnTruncateTask.java| 5 +++--
.../ql/parse/spark/SparkPartitionPruningSinkOperator.java | 5 +++--
6 files changed, 27 insertions(+), 15 deletions(-)
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/exec/SparkHashTableSinkOperator.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/SparkHashTableSinkOperator.java
index 78ae9a1..10144a1 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/SparkHashTableSinkOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/SparkHashTableSinkOperator.java
@@ -22,6 +22,7 @@ import java.io.ObjectOutputStream;
import java.io.OutputStream;
import java.io.Serializable;
import java.util.Set;
+import java.util.concurrent.ThreadLocalRandom;
import org.apache.commons.io.FileExistsException;
import org.apache.hadoop.conf.Configuration;
@@ -140,7 +141,7 @@ public class SparkHashTableSinkOperator
fs.mkdirs(path); // Create the folder and its parents if not there
while (true) {
path = new Path(path, getOperatorId()
-+ "-" + Math.abs(Utilities.randGen.nextInt()));
++ "-" + Math.abs(ThreadLocalRandom.current().nextInt()));
try {
// This will guarantee file name uniqueness.
if (fs.createNewFile(path)) {
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
index 7fd42c1..a7770b4 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
@@ -67,6 +67,7 @@ import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
+import java.util.concurrent.ThreadLocalRandom;
import java.util.concurrent.atomic.AtomicLong;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -257,8 +258,6 @@ public final class Utilities {
@Deprecated
protected static final String DEPRECATED_MAPRED_DFSCLIENT_PARALLELISM_MAX =
"mapred.dfsclient.parallelism.max";
- public static Random randGen = new Random();
-
private static final Object INPUT_SUMMARY_LOCK = new Object();
private static final Object ROOT_HDFS_DIR_LOCK = new Object();
@@ -751,7 +750,8 @@ public final class Utilities {
public static String getTaskId(Configuration hconf) {
String taskid = (hconf == null) ? null : hconf.get("mapred.task.id");
if (StringUtils.isEmpty(taskid)) {
- return (Integer.toString(randGen.nextInt(Integer.MAX_VALUE)));
+ return (Integer
+ .toString(ThreadLocalRandom.current().nextInt(Integer.MAX_VALUE)));
} else {
/*
* extract the task and attempt id from the hadoop taskid. in version 17
the leading component
@@ -2894,7 +2894,8 @@ public final class Utilities {
if (failures >= maxRetries) {
throw e;
}
-long waitTime = getRandomWaitTime(baseWindow, failures, randGen);
+long waitTime = getRandomWaitTime(baseWindow, failures,
+ThreadLocalRandom.current());
try {
Thread.sleep(waitTime);
} catch (InterruptedException iex) {
@@ -2933,7 +2934,8 @@ public final class Utilities {
LOG.error("Error during JDBC connection.", e);
throw e;
}
-long waitTime = Utilities.getRandomWaitTime(waitWindow, failures,
randGen);
+long waitTime = Utilities.getRandomWaitTime(waitWindow, failures,
+ThreadLocalRandom.current());
try {
Thread.sleep(waitTime);
} catch (InterruptedException e1) {
@@ -2972,7 +2974,8 @@ public final class Utilities {
LOG.error("Error preparing JDBC Statement {}", stmt, e);
throw e;
}
-long waitTime = Utilities.getRandomWaitTime(waitWindow, failures,
randGen);
+long waitTime = Utilities.getRandomWaitTime(waitWindow, failures,
+ThreadLocalRandom.current());
try {
Thread.sleep(waitTime);
} catch (InterruptedException e1) {
diff --git
[hive] branch master updated: HIVE-22240: Function percentile_cont fails when array parameter passed (Krisztian Kasa, reviewed by Jesus Camacho Rodriguez)
This is an automated email from the ASF dual-hosted git repository.
jcamacho pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 72094da HIVE-22240: Function percentile_cont fails when array
parameter passed (Krisztian Kasa, reviewed by Jesus Camacho Rodriguez)
72094da is described below
commit 72094da54622f27215ee88878fbe78161e2757bb
Author: Krisztian Kasa
AuthorDate: Mon Oct 21 12:35:11 2019 -0700
HIVE-22240: Function percentile_cont fails when array parameter passed
(Krisztian Kasa, reviewed by Jesus Camacho Rodriguez)
---
.../ql/udf/generic/GenericUDAFPercentileCont.java | 132 +--
.../ql/udf/generic/GenericUDAFPercentileDisc.java | 41 +-
.../queries/clientpositive/udaf_percentile_cont.q | 14 +-
.../queries/clientpositive/udaf_percentile_disc.q | 14 +-
.../clientpositive/udaf_percentile_cont.q.out | 424 +++--
.../clientpositive/udaf_percentile_disc.q.out | 424 +++--
6 files changed, 596 insertions(+), 453 deletions(-)
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFPercentileCont.java
b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFPercentileCont.java
index 00e17f9..ad61410 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFPercentileCont.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFPercentileCont.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.hive.ql.udf.generic;
+import static java.util.Collections.singletonList;
import static org.apache.hadoop.hive.ql.util.DirectionUtils.DESCENDING_CODE;
import java.io.Serializable;
@@ -88,11 +89,13 @@ public class GenericUDAFPercentileCont extends
AbstractGenericUDAFResolver {
case INT:
case LONG:
case VOID:
- return new PercentileContLongEvaluator();
+ return parameters[1].getCategory() == ObjectInspector.Category.LIST ?
+ new PercentileContLongArrayEvaluator() : new
PercentileContLongEvaluator();
case FLOAT:
case DOUBLE:
case DECIMAL:
- return new PercentileContDoubleEvaluator();
+ return parameters[1].getCategory() == ObjectInspector.Category.LIST ?
+ new PercentileContDoubleArrayEvaluator() : new
PercentileContDoubleEvaluator();
case STRING:
case TIMESTAMP:
case VARCHAR:
@@ -155,8 +158,9 @@ public class GenericUDAFPercentileCont extends
AbstractGenericUDAFResolver {
public abstract static class PercentileContEvaluator extends
GenericUDAFEvaluator {
PercentileCalculator calc = getCalculator();
-protected PercentileContEvaluator(Comparator>
comparator) {
+protected PercentileContEvaluator(Comparator>
comparator, Converter converter) {
this.comparator = comparator;
+ this.converter = converter;
}
/**
@@ -177,7 +181,7 @@ public class GenericUDAFPercentileCont extends
AbstractGenericUDAFResolver {
protected transient Object[] partialResult;
// FINAL and COMPLETE output
-protected DoubleWritable result;
+protected List results;
// PARTIAL2 and FINAL inputs
protected transient StructObjectInspector soi;
@@ -186,6 +190,7 @@ public class GenericUDAFPercentileCont extends
AbstractGenericUDAFResolver {
protected transient StructField isAscendingField;
private final transient Comparator> comparator;
+private final transient Converter converter;
protected transient boolean isAscending;
public ObjectInspector init(Mode m, ObjectInspector[] parameters) throws
HiveException {
@@ -205,8 +210,8 @@ public class GenericUDAFPercentileCont extends
AbstractGenericUDAFResolver {
return ObjectInspectorFactory.getStandardStructObjectInspector(fname,
foi);
} else { // ...for final result
-result = new DoubleWritable(0);
-return PrimitiveObjectInspectorFactory.writableDoubleObjectInspector;
+results = null;
+return converter.getResultObjectInspector();
}
}
@@ -263,12 +268,9 @@ public class GenericUDAFPercentileCont extends
AbstractGenericUDAFResolver {
@Override
public void iterate(AggregationBuffer agg, Object[] parameters) throws
HiveException {
PercentileAgg percAgg = (PercentileAgg) agg;
- Double percentile = ((HiveDecimalWritable)
parameters[1]).getHiveDecimal().doubleValue();
if (percAgg.percentiles == null) {
-validatePercentile(percentile);
-percAgg.percentiles = new ArrayList(1);
-percAgg.percentiles.add(new DoubleWritable(percentile));
+percAgg.percentiles =
converter.convertPercentileParameter(parameters[1]);
}
if (parameters[0] == null) {
@@ -341,15 +343,14 @@ public class GenericUDAFPercentileCont extends
AbstractGenericUDAFResolver {
// Accumulate the counts.
long total = getTotal(entriesList);
-
- // Initialize
[hive] branch master updated: HIVE-21246: Un-bury DelimitedJSONSerDe from PlanUtils.java (David Mollitor, reviewed by Laszlo Bodor)
This is an automated email from the ASF dual-hosted git repository.
dmollitor pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new c9850b4 HIVE-21246: Un-bury DelimitedJSONSerDe from PlanUtils.java
(David Mollitor, reviewed by Laszlo Bodor)
c9850b4 is described below
commit c9850b443de4e38903ae4c91a9619a71b2f3db33
Author: David Mollitor
AuthorDate: Mon Oct 21 09:26:06 2019 -0400
HIVE-21246: Un-bury DelimitedJSONSerDe from PlanUtils.java (David Mollitor,
reviewed by Laszlo Bodor)
---
.../hadoop/hive/ql/parse/SemanticAnalyzer.java | 9 --
.../org/apache/hadoop/hive/ql/plan/PlanUtils.java | 34 +-
2 files changed, 15 insertions(+), 28 deletions(-)
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
index 281025f..30d3791 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
@@ -257,6 +257,7 @@ import
org.apache.hadoop.hive.ql.udf.generic.GenericUDTFInline;
import org.apache.hadoop.hive.ql.util.DirectionUtils;
import org.apache.hadoop.hive.ql.util.ResourceDownloader;
import org.apache.hadoop.hive.serde.serdeConstants;
+import org.apache.hadoop.hive.serde2.DelimitedJSONSerDe;
import org.apache.hadoop.hive.serde2.Deserializer;
import org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe;
import org.apache.hadoop.hive.serde2.NoOpFetchFormatter;
@@ -4010,9 +4011,13 @@ public class SemanticAnalyzer extends
BaseSemanticAnalyzer {
.getChild(inputSerDeNum))).getChild(0), inpColumns.toString(),
inpColumnTypes.toString());
} else {
- inInfo = PlanUtils.getTableDesc(serde, Integer
+ // It is not a very clean way, and should be modified later - due to
+ // compatibility reasons, user sees the results as JSON for custom
+ // scripts and has no way for specifying that. Right now, it is
+ // hard-coded to DelimitedJSONSerDe
+ inInfo = PlanUtils.getTableDesc(DelimitedJSONSerDe.class, Integer
.toString(fieldSeparator), inpColumns.toString(), inpColumnTypes
- .toString(), false, true);
+ .toString(), false);
}
if (trfm.getChild(outputSerDeNum).getChildCount() > 0) {
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java
b/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java
index 563d053..e20f695 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java
@@ -67,7 +67,6 @@ import org.apache.hadoop.hive.ql.parse.TypeCheckProcFactory;
import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hadoop.hive.serde.serdeConstants;
import org.apache.hadoop.hive.serde2.AbstractSerDe;
-import org.apache.hadoop.hive.serde2.DelimitedJSONSerDe;
import org.apache.hadoop.hive.serde2.Deserializer;
import org.apache.hadoop.hive.serde2.binarysortable.BinarySortableSerDe;
import org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe;
@@ -234,26 +233,18 @@ public final class PlanUtils {
columnTypes, lastColumnTakesRestOfTheLine);
}
- public static TableDesc getTableDesc(
- Class serdeClass, String separatorCode,
- String columns, String columnTypes, boolean
lastColumnTakesRestOfTheLine) {
-return getTableDesc(serdeClass, separatorCode, columns, columnTypes,
-lastColumnTakesRestOfTheLine, false);
- }
-
- public static TableDesc getTableDesc(
- Class serdeClass, String separatorCode,
- String columns, String columnTypes, boolean lastColumnTakesRestOfTheLine,
- boolean useDelimitedJSON) {
+ public static TableDesc getTableDesc(Class
serdeClass,
+ String separatorCode, String columns, String columnTypes,
+ boolean lastColumnTakesRestOfTheLine) {
return getTableDesc(serdeClass, separatorCode, columns, columnTypes,
-lastColumnTakesRestOfTheLine, useDelimitedJSON, "TextFile");
+lastColumnTakesRestOfTheLine, "TextFile");
}
public static TableDesc getTableDesc(
Class serdeClass, String separatorCode,
String columns, String columnTypes, boolean lastColumnTakesRestOfTheLine,
- boolean useDelimitedJSON, String fileFormat) {
+ String fileFormat) {
Properties properties = Utilities.makeProperties(
serdeConstants.SERIALIZATION_FORMAT, separatorCode,
serdeConstants.LIST_COLUMNS,
@@ -272,15 +263,6 @@ public final class PlanUtils {
"true");
}
-// It is not a very clean way, and should be modified later - due to
-// compatibility reasons,
-// user sees the results as json for custom scripts and has no way for
-// specifying that.
-// Right now, it is hard-coded in the code
-if (useDelimitedJSON) {
-
[hive] branch master updated: HIVE-22346: Yetus is failing rat check (Krisztian Kasa via Zoltan Haindrich)
This is an automated email from the ASF dual-hosted git repository. kgyrtkirk pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/hive.git The following commit(s) were added to refs/heads/master by this push: new 1866d7d HIVE-22346: Yetus is failing rat check (Krisztian Kasa via Zoltan Haindrich) 1866d7d is described below commit 1866d7d4c74bad4eb0e2cedf5e5d3ce342ac8957 Author: Krisztian Kasa AuthorDate: Mon Oct 21 10:42:29 2019 +0200 HIVE-22346: Yetus is failing rat check (Krisztian Kasa via Zoltan Haindrich) Signed-off-by: Zoltan Haindrich --- pom.xml | 9 + standalone-metastore/metastore-server/pom.xml | 1 + 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/pom.xml b/pom.xml index ff2c86a..6dbff13 100644 --- a/pom.xml +++ b/pom.xml @@ -1354,8 +1354,8 @@ checkstyle/** bin/** itests/** -**/README.md -**/*.iml + **/README.md + **/*.iml **/*.txt **/*.log **/*.arcconfig @@ -1383,8 +1383,9 @@ **/*.html **/sit **/test/queries/**/*.sql -**/patchprocess/** -**/metastore_db/** + **/patchprocess/** + **/metastore_db/** + **/test/resources/**/*.ldif diff --git a/standalone-metastore/metastore-server/pom.xml b/standalone-metastore/metastore-server/pom.xml index 7645a98..b86acb6 100644 --- a/standalone-metastore/metastore-server/pom.xml +++ b/standalone-metastore/metastore-server/pom.xml @@ -605,6 +605,7 @@ **/gen/** **/patchprocess/** **/metastore_db/** +**/test/resources/**/*.ldif
