This is an automated email from the ASF dual-hosted git repository. sorabh pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/drill.git
commit e191c54cab9ef969e290dc2bf95e100cb4f5b1eb Author: Aman Sinha <[email protected]> AuthorDate: Tue Apr 30 12:03:53 2019 -0700 DRILL-7228: Upgrade to a newer version of t-digest to address inaccuracies in histogram buckets. closes #1774 --- exec/java-exec/pom.xml | 5 + .../drill/exec/expr/fn/impl/TDigestFunctions.java | 140 ++++++++++----------- .../impl/statistics/TDigestMergedStatistic.java | 12 +- .../planner/common/NumericEquiDepthHistogram.java | 4 +- 4 files changed, 83 insertions(+), 78 deletions(-) diff --git a/exec/java-exec/pom.xml b/exec/java-exec/pom.xml index 2fcd935..c35e935 100644 --- a/exec/java-exec/pom.xml +++ b/exec/java-exec/pom.xml @@ -344,6 +344,11 @@ <version>2.7.0</version> </dependency> <dependency> + <groupId>com.tdunning</groupId> + <artifactId>t-digest</artifactId> + <version>3.2</version> + </dependency> + <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-common</artifactId> <exclusions> diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/TDigestFunctions.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/TDigestFunctions.java index 041543b..bfcf78d 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/TDigestFunctions.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/TDigestFunctions.java @@ -67,13 +67,13 @@ public class TDigestFunctions { public void setup() { work = new ObjectHolder(); compression.value = (int) options.getLong(org.apache.drill.exec.ExecConstants.TDIGEST_COMPRESSION); - work.obj = new com.clearspring.analytics.stream.quantile.TDigest(compression.value); + work.obj = new com.tdunning.math.stats.MergingDigest(compression.value); } @Override public void add() { if (work.obj != null) { - com.clearspring.analytics.stream.quantile.TDigest tdigest = (com.clearspring.analytics.stream.quantile.TDigest) work.obj; + com.tdunning.math.stats.MergingDigest tdigest = (com.tdunning.math.stats.MergingDigest) work.obj; tdigest.add(in.value); } } @@ -81,7 +81,7 @@ public class TDigestFunctions { @Override public void output() { if (work.obj != null) { - com.clearspring.analytics.stream.quantile.TDigest tdigest = (com.clearspring.analytics.stream.quantile.TDigest) work.obj; + com.tdunning.math.stats.MergingDigest tdigest = (com.tdunning.math.stats.MergingDigest) work.obj; try { if (tdigest.size() > 0) { int size = tdigest.smallByteSize(); @@ -105,7 +105,7 @@ public class TDigestFunctions { @Override public void reset() { - work.obj = new com.clearspring.analytics.stream.quantile.TDigest(compression.value); + work.obj = new com.tdunning.math.stats.MergingDigest(compression.value); } } @@ -122,13 +122,13 @@ public class TDigestFunctions { public void setup() { work = new ObjectHolder(); compression.value = (int) options.getLong(org.apache.drill.exec.ExecConstants.TDIGEST_COMPRESSION); - work.obj = new com.clearspring.analytics.stream.quantile.TDigest(compression.value); + work.obj = new com.tdunning.math.stats.MergingDigest(compression.value); } @Override public void add() { if (work.obj != null) { - com.clearspring.analytics.stream.quantile.TDigest tdigest = (com.clearspring.analytics.stream.quantile.TDigest) work.obj; + com.tdunning.math.stats.MergingDigest tdigest = (com.tdunning.math.stats.MergingDigest) work.obj; if (in.isSet == 1) { tdigest.add(in.value); } else { @@ -140,7 +140,7 @@ public class TDigestFunctions { @Override public void output() { if (work.obj != null) { - com.clearspring.analytics.stream.quantile.TDigest tdigest = (com.clearspring.analytics.stream.quantile.TDigest) work.obj; + com.tdunning.math.stats.MergingDigest tdigest = (com.tdunning.math.stats.MergingDigest) work.obj; try { if (tdigest.size() > 0) { int size = tdigest.smallByteSize(); @@ -164,7 +164,7 @@ public class TDigestFunctions { @Override public void reset() { - work.obj = new com.clearspring.analytics.stream.quantile.TDigest(compression.value); + work.obj = new com.tdunning.math.stats.MergingDigest(compression.value); } } @@ -181,13 +181,13 @@ public class TDigestFunctions { public void setup() { work = new ObjectHolder(); compression.value = (int) options.getLong(org.apache.drill.exec.ExecConstants.TDIGEST_COMPRESSION); - work.obj = new com.clearspring.analytics.stream.quantile.TDigest(compression.value); + work.obj = new com.tdunning.math.stats.MergingDigest(compression.value); } @Override public void add() { if (work.obj != null) { - com.clearspring.analytics.stream.quantile.TDigest tdigest = (com.clearspring.analytics.stream.quantile.TDigest) work.obj; + com.tdunning.math.stats.MergingDigest tdigest = (com.tdunning.math.stats.MergingDigest) work.obj; tdigest.add(in.value); } } @@ -195,7 +195,7 @@ public class TDigestFunctions { @Override public void output() { if (work.obj != null) { - com.clearspring.analytics.stream.quantile.TDigest tdigest = (com.clearspring.analytics.stream.quantile.TDigest) work.obj; + com.tdunning.math.stats.MergingDigest tdigest = (com.tdunning.math.stats.MergingDigest) work.obj; try { if (tdigest.size() > 0) { int size = tdigest.smallByteSize(); @@ -219,7 +219,7 @@ public class TDigestFunctions { @Override public void reset() { - work.obj = new com.clearspring.analytics.stream.quantile.TDigest(compression.value); + work.obj = new com.tdunning.math.stats.MergingDigest(compression.value); } } @@ -236,13 +236,13 @@ public class TDigestFunctions { public void setup() { work = new ObjectHolder(); compression.value = (int) options.getLong(org.apache.drill.exec.ExecConstants.TDIGEST_COMPRESSION); - work.obj = new com.clearspring.analytics.stream.quantile.TDigest(compression.value); + work.obj = new com.tdunning.math.stats.MergingDigest(compression.value); } @Override public void add() { if (work.obj != null) { - com.clearspring.analytics.stream.quantile.TDigest tdigest = (com.clearspring.analytics.stream.quantile.TDigest) work.obj; + com.tdunning.math.stats.MergingDigest tdigest = (com.tdunning.math.stats.MergingDigest) work.obj; if (in.isSet == 1) { tdigest.add(in.value); } else { @@ -254,7 +254,7 @@ public class TDigestFunctions { @Override public void output() { if (work.obj != null) { - com.clearspring.analytics.stream.quantile.TDigest tdigest = (com.clearspring.analytics.stream.quantile.TDigest) work.obj; + com.tdunning.math.stats.MergingDigest tdigest = (com.tdunning.math.stats.MergingDigest) work.obj; try { if (tdigest.size() > 0) { int size = tdigest.smallByteSize(); @@ -278,7 +278,7 @@ public class TDigestFunctions { @Override public void reset() { - work.obj = new com.clearspring.analytics.stream.quantile.TDigest(compression.value); + work.obj = new com.tdunning.math.stats.MergingDigest(compression.value); } } @@ -295,13 +295,13 @@ public class TDigestFunctions { public void setup() { work = new ObjectHolder(); compression.value = (int) options.getLong(org.apache.drill.exec.ExecConstants.TDIGEST_COMPRESSION); - work.obj = new com.clearspring.analytics.stream.quantile.TDigest(compression.value); + work.obj = new com.tdunning.math.stats.MergingDigest(compression.value); } @Override public void add() { if (work.obj != null) { - com.clearspring.analytics.stream.quantile.TDigest tdigest = (com.clearspring.analytics.stream.quantile.TDigest) work.obj; + com.tdunning.math.stats.MergingDigest tdigest = (com.tdunning.math.stats.MergingDigest) work.obj; tdigest.add(in.value); } } @@ -309,7 +309,7 @@ public class TDigestFunctions { @Override public void output() { if (work.obj != null) { - com.clearspring.analytics.stream.quantile.TDigest tdigest = (com.clearspring.analytics.stream.quantile.TDigest) work.obj; + com.tdunning.math.stats.MergingDigest tdigest = (com.tdunning.math.stats.MergingDigest) work.obj; try { if (tdigest.size() > 0) { int size = tdigest.smallByteSize(); @@ -333,7 +333,7 @@ public class TDigestFunctions { @Override public void reset() { - work.obj = new com.clearspring.analytics.stream.quantile.TDigest(compression.value); + work.obj = new com.tdunning.math.stats.MergingDigest(compression.value); } } @@ -350,13 +350,13 @@ public class TDigestFunctions { public void setup() { work = new ObjectHolder(); compression.value = (int) options.getLong(org.apache.drill.exec.ExecConstants.TDIGEST_COMPRESSION); - work.obj = new com.clearspring.analytics.stream.quantile.TDigest(compression.value); + work.obj = new com.tdunning.math.stats.MergingDigest(compression.value); } @Override public void add() { if (work.obj != null) { - com.clearspring.analytics.stream.quantile.TDigest tdigest = (com.clearspring.analytics.stream.quantile.TDigest) work.obj; + com.tdunning.math.stats.MergingDigest tdigest = (com.tdunning.math.stats.MergingDigest) work.obj; if (in.isSet == 1) { tdigest.add(in.value); } else { @@ -368,7 +368,7 @@ public class TDigestFunctions { @Override public void output() { if (work.obj != null) { - com.clearspring.analytics.stream.quantile.TDigest tdigest = (com.clearspring.analytics.stream.quantile.TDigest) work.obj; + com.tdunning.math.stats.MergingDigest tdigest = (com.tdunning.math.stats.MergingDigest) work.obj; try { if (tdigest.size() > 0) { int size = tdigest.smallByteSize(); @@ -392,7 +392,7 @@ public class TDigestFunctions { @Override public void reset() { - work.obj = new com.clearspring.analytics.stream.quantile.TDigest(compression.value); + work.obj = new com.tdunning.math.stats.MergingDigest(compression.value); } } @@ -409,13 +409,13 @@ public class TDigestFunctions { public void setup() { work = new ObjectHolder(); compression.value = (int) options.getLong(org.apache.drill.exec.ExecConstants.TDIGEST_COMPRESSION); - work.obj = new com.clearspring.analytics.stream.quantile.TDigest(compression.value); + work.obj = new com.tdunning.math.stats.MergingDigest(compression.value); } @Override public void add() { if (work.obj != null) { - com.clearspring.analytics.stream.quantile.TDigest tdigest = (com.clearspring.analytics.stream.quantile.TDigest) work.obj; + com.tdunning.math.stats.MergingDigest tdigest = (com.tdunning.math.stats.MergingDigest) work.obj; tdigest.add(in.value); } } @@ -423,7 +423,7 @@ public class TDigestFunctions { @Override public void output() { if (work.obj != null) { - com.clearspring.analytics.stream.quantile.TDigest tdigest = (com.clearspring.analytics.stream.quantile.TDigest) work.obj; + com.tdunning.math.stats.MergingDigest tdigest = (com.tdunning.math.stats.MergingDigest) work.obj; try { if (tdigest.size() > 0) { int size = tdigest.smallByteSize(); @@ -447,7 +447,7 @@ public class TDigestFunctions { @Override public void reset() { - work.obj = new com.clearspring.analytics.stream.quantile.TDigest(compression.value); + work.obj = new com.tdunning.math.stats.MergingDigest(compression.value); } } @@ -464,13 +464,13 @@ public class TDigestFunctions { public void setup() { work = new ObjectHolder(); compression.value = (int) options.getLong(org.apache.drill.exec.ExecConstants.TDIGEST_COMPRESSION); - work.obj = new com.clearspring.analytics.stream.quantile.TDigest(compression.value); + work.obj = new com.tdunning.math.stats.MergingDigest(compression.value); } @Override public void add() { if (work.obj != null) { - com.clearspring.analytics.stream.quantile.TDigest tdigest = (com.clearspring.analytics.stream.quantile.TDigest) work.obj; + com.tdunning.math.stats.MergingDigest tdigest = (com.tdunning.math.stats.MergingDigest) work.obj; if (in.isSet == 1) { tdigest.add(in.value); } else { @@ -482,7 +482,7 @@ public class TDigestFunctions { @Override public void output() { if (work.obj != null) { - com.clearspring.analytics.stream.quantile.TDigest tdigest = (com.clearspring.analytics.stream.quantile.TDigest) work.obj; + com.tdunning.math.stats.MergingDigest tdigest = (com.tdunning.math.stats.MergingDigest) work.obj; try { if (tdigest.size() > 0) { int size = tdigest.smallByteSize(); @@ -506,7 +506,7 @@ public class TDigestFunctions { @Override public void reset() { - work.obj = new com.clearspring.analytics.stream.quantile.TDigest(compression.value); + work.obj = new com.tdunning.math.stats.MergingDigest(compression.value); } } @@ -523,13 +523,13 @@ public class TDigestFunctions { public void setup() { work = new ObjectHolder(); compression.value = (int) options.getLong(org.apache.drill.exec.ExecConstants.TDIGEST_COMPRESSION); - work.obj = new com.clearspring.analytics.stream.quantile.TDigest(compression.value); + work.obj = new com.tdunning.math.stats.MergingDigest(compression.value); } @Override public void add() { if (work.obj != null) { - com.clearspring.analytics.stream.quantile.TDigest tdigest = (com.clearspring.analytics.stream.quantile.TDigest) work.obj; + com.tdunning.math.stats.MergingDigest tdigest = (com.tdunning.math.stats.MergingDigest) work.obj; tdigest.add(in.value); } } @@ -537,7 +537,7 @@ public class TDigestFunctions { @Override public void output() { if (work.obj != null) { - com.clearspring.analytics.stream.quantile.TDigest tdigest = (com.clearspring.analytics.stream.quantile.TDigest) work.obj; + com.tdunning.math.stats.MergingDigest tdigest = (com.tdunning.math.stats.MergingDigest) work.obj; try { if (tdigest.size() > 0) { int size = tdigest.smallByteSize(); @@ -561,7 +561,7 @@ public class TDigestFunctions { @Override public void reset() { - work.obj = new com.clearspring.analytics.stream.quantile.TDigest(compression.value); + work.obj = new com.tdunning.math.stats.MergingDigest(compression.value); } } @@ -578,13 +578,13 @@ public class TDigestFunctions { public void setup() { work = new ObjectHolder(); compression.value = (int) options.getLong(org.apache.drill.exec.ExecConstants.TDIGEST_COMPRESSION); - work.obj = new com.clearspring.analytics.stream.quantile.TDigest(compression.value); + work.obj = new com.tdunning.math.stats.MergingDigest(compression.value); } @Override public void add() { if (work.obj != null) { - com.clearspring.analytics.stream.quantile.TDigest tdigest = (com.clearspring.analytics.stream.quantile.TDigest) work.obj; + com.tdunning.math.stats.MergingDigest tdigest = (com.tdunning.math.stats.MergingDigest) work.obj; if (in.isSet == 1) { tdigest.add(in.value); } else { @@ -596,7 +596,7 @@ public class TDigestFunctions { @Override public void output() { if (work.obj != null) { - com.clearspring.analytics.stream.quantile.TDigest tdigest = (com.clearspring.analytics.stream.quantile.TDigest) work.obj; + com.tdunning.math.stats.MergingDigest tdigest = (com.tdunning.math.stats.MergingDigest) work.obj; try { if (tdigest.size() > 0) { int size = tdigest.smallByteSize(); @@ -620,7 +620,7 @@ public class TDigestFunctions { @Override public void reset() { - work.obj = new com.clearspring.analytics.stream.quantile.TDigest(compression.value); + work.obj = new com.tdunning.math.stats.MergingDigest(compression.value); } } @@ -637,13 +637,13 @@ public class TDigestFunctions { public void setup() { work = new ObjectHolder(); compression.value = (int) options.getLong(org.apache.drill.exec.ExecConstants.TDIGEST_COMPRESSION); - work.obj = new com.clearspring.analytics.stream.quantile.TDigest(compression.value); + work.obj = new com.tdunning.math.stats.MergingDigest(compression.value); } @Override public void add() { if (work.obj != null) { - com.clearspring.analytics.stream.quantile.TDigest tdigest = (com.clearspring.analytics.stream.quantile.TDigest) work.obj; + com.tdunning.math.stats.MergingDigest tdigest = (com.tdunning.math.stats.MergingDigest) work.obj; tdigest.add(in.value); } } @@ -651,7 +651,7 @@ public class TDigestFunctions { @Override public void output() { if (work.obj != null) { - com.clearspring.analytics.stream.quantile.TDigest tdigest = (com.clearspring.analytics.stream.quantile.TDigest) work.obj; + com.tdunning.math.stats.MergingDigest tdigest = (com.tdunning.math.stats.MergingDigest) work.obj; try { if (tdigest.size() > 0) { int size = tdigest.smallByteSize(); @@ -675,7 +675,7 @@ public class TDigestFunctions { @Override public void reset() { - work.obj = new com.clearspring.analytics.stream.quantile.TDigest(compression.value); + work.obj = new com.tdunning.math.stats.MergingDigest(compression.value); } } @@ -692,13 +692,13 @@ public class TDigestFunctions { public void setup() { work = new ObjectHolder(); compression.value = (int) options.getLong(org.apache.drill.exec.ExecConstants.TDIGEST_COMPRESSION); - work.obj = new com.clearspring.analytics.stream.quantile.TDigest(compression.value); + work.obj = new com.tdunning.math.stats.MergingDigest(compression.value); } @Override public void add() { if (work.obj != null) { - com.clearspring.analytics.stream.quantile.TDigest tdigest = (com.clearspring.analytics.stream.quantile.TDigest) work.obj; + com.tdunning.math.stats.MergingDigest tdigest = (com.tdunning.math.stats.MergingDigest) work.obj; if (in.isSet == 1) { tdigest.add(in.value); } else { @@ -710,7 +710,7 @@ public class TDigestFunctions { @Override public void output() { if (work.obj != null) { - com.clearspring.analytics.stream.quantile.TDigest tdigest = (com.clearspring.analytics.stream.quantile.TDigest) work.obj; + com.tdunning.math.stats.MergingDigest tdigest = (com.tdunning.math.stats.MergingDigest) work.obj; try { if (tdigest.size() > 0) { int size = tdigest.smallByteSize(); @@ -734,7 +734,7 @@ public class TDigestFunctions { @Override public void reset() { - work.obj = new com.clearspring.analytics.stream.quantile.TDigest(compression.value); + work.obj = new com.tdunning.math.stats.MergingDigest(compression.value); } } @@ -751,13 +751,13 @@ public class TDigestFunctions { public void setup() { work = new ObjectHolder(); compression.value = (int) options.getLong(org.apache.drill.exec.ExecConstants.TDIGEST_COMPRESSION); - work.obj = new com.clearspring.analytics.stream.quantile.TDigest(compression.value); + work.obj = new com.tdunning.math.stats.MergingDigest(compression.value); } @Override public void add() { if (work.obj != null) { - com.clearspring.analytics.stream.quantile.TDigest tdigest = (com.clearspring.analytics.stream.quantile.TDigest) work.obj; + com.tdunning.math.stats.MergingDigest tdigest = (com.tdunning.math.stats.MergingDigest) work.obj; tdigest.add(in.value); } } @@ -765,7 +765,7 @@ public class TDigestFunctions { @Override public void output() { if (work.obj != null) { - com.clearspring.analytics.stream.quantile.TDigest tdigest = (com.clearspring.analytics.stream.quantile.TDigest) work.obj; + com.tdunning.math.stats.MergingDigest tdigest = (com.tdunning.math.stats.MergingDigest) work.obj; try { if (tdigest.size() > 0) { int size = tdigest.smallByteSize(); @@ -789,7 +789,7 @@ public class TDigestFunctions { @Override public void reset() { - work.obj = new com.clearspring.analytics.stream.quantile.TDigest(compression.value); + work.obj = new com.tdunning.math.stats.MergingDigest(compression.value); } } @@ -806,13 +806,13 @@ public class TDigestFunctions { public void setup() { work = new ObjectHolder(); compression.value = (int) options.getLong(org.apache.drill.exec.ExecConstants.TDIGEST_COMPRESSION); - work.obj = new com.clearspring.analytics.stream.quantile.TDigest(compression.value); + work.obj = new com.tdunning.math.stats.MergingDigest(compression.value); } @Override public void add() { if (work.obj != null) { - com.clearspring.analytics.stream.quantile.TDigest tdigest = (com.clearspring.analytics.stream.quantile.TDigest) work.obj; + com.tdunning.math.stats.MergingDigest tdigest = (com.tdunning.math.stats.MergingDigest) work.obj; if (in.isSet == 1) { tdigest.add(in.value); } else { @@ -824,7 +824,7 @@ public class TDigestFunctions { @Override public void output() { if (work.obj != null) { - com.clearspring.analytics.stream.quantile.TDigest tdigest = (com.clearspring.analytics.stream.quantile.TDigest) work.obj; + com.tdunning.math.stats.MergingDigest tdigest = (com.tdunning.math.stats.MergingDigest) work.obj; try { if (tdigest.size() > 0) { int size = tdigest.smallByteSize(); @@ -848,7 +848,7 @@ public class TDigestFunctions { @Override public void reset() { - work.obj = new com.clearspring.analytics.stream.quantile.TDigest(compression.value); + work.obj = new com.tdunning.math.stats.MergingDigest(compression.value); } } @@ -865,13 +865,13 @@ public class TDigestFunctions { public void setup() { work = new ObjectHolder(); compression.value = (int) options.getLong(org.apache.drill.exec.ExecConstants.TDIGEST_COMPRESSION); - work.obj = new com.clearspring.analytics.stream.quantile.TDigest(compression.value); + work.obj = new com.tdunning.math.stats.MergingDigest(compression.value); } @Override public void add() { if (work.obj != null) { - com.clearspring.analytics.stream.quantile.TDigest tdigest = (com.clearspring.analytics.stream.quantile.TDigest) work.obj; + com.tdunning.math.stats.MergingDigest tdigest = (com.tdunning.math.stats.MergingDigest) work.obj; tdigest.add(in.value); } } @@ -879,7 +879,7 @@ public class TDigestFunctions { @Override public void output() { if (work.obj != null) { - com.clearspring.analytics.stream.quantile.TDigest tdigest = (com.clearspring.analytics.stream.quantile.TDigest) work.obj; + com.tdunning.math.stats.MergingDigest tdigest = (com.tdunning.math.stats.MergingDigest) work.obj; try { if (tdigest.size() > 0) { int size = tdigest.smallByteSize(); @@ -903,7 +903,7 @@ public class TDigestFunctions { @Override public void reset() { - work.obj = new com.clearspring.analytics.stream.quantile.TDigest(compression.value); + work.obj = new com.tdunning.math.stats.MergingDigest(compression.value); } } @@ -920,13 +920,13 @@ public class TDigestFunctions { public void setup() { work = new ObjectHolder(); compression.value = (int) options.getLong(org.apache.drill.exec.ExecConstants.TDIGEST_COMPRESSION); - work.obj = new com.clearspring.analytics.stream.quantile.TDigest(compression.value); + work.obj = new com.tdunning.math.stats.MergingDigest(compression.value); } @Override public void add() { if (work.obj != null) { - com.clearspring.analytics.stream.quantile.TDigest tdigest = (com.clearspring.analytics.stream.quantile.TDigest) work.obj; + com.tdunning.math.stats.MergingDigest tdigest = (com.tdunning.math.stats.MergingDigest) work.obj; if (in.isSet == 1) { tdigest.add(in.value); } else { @@ -938,7 +938,7 @@ public class TDigestFunctions { @Override public void output() { if (work.obj != null) { - com.clearspring.analytics.stream.quantile.TDigest tdigest = (com.clearspring.analytics.stream.quantile.TDigest) work.obj; + com.tdunning.math.stats.MergingDigest tdigest = (com.tdunning.math.stats.MergingDigest) work.obj; try { if (tdigest.size() > 0) { int size = tdigest.smallByteSize(); @@ -962,7 +962,7 @@ public class TDigestFunctions { @Override public void reset() { - work.obj = new com.clearspring.analytics.stream.quantile.TDigest(compression.value); + work.obj = new com.tdunning.math.stats.MergingDigest(compression.value); } } @@ -1097,18 +1097,18 @@ public class TDigestFunctions { public void setup() { work = new ObjectHolder(); compression.value = (int) options.getLong(org.apache.drill.exec.ExecConstants.TDIGEST_COMPRESSION); - work.obj = new com.clearspring.analytics.stream.quantile.TDigest(compression.value); + work.obj = new com.tdunning.math.stats.MergingDigest(compression.value); } @Override public void add() { if (work.obj != null) { - com.clearspring.analytics.stream.quantile.TDigest tdigest = (com.clearspring.analytics.stream.quantile.TDigest) work.obj; + com.tdunning.math.stats.MergingDigest tdigest = (com.tdunning.math.stats.MergingDigest) work.obj; try { if (in.isSet != 0) { byte[] buf = org.apache.drill.exec.expr.fn.impl.StringFunctionHelpers.toStringFromUTF8(in.start, in.end, in.buffer).getBytes(); - com.clearspring.analytics.stream.quantile.TDigest other = - com.clearspring.analytics.stream.quantile.TDigest.fromBytes(java.nio.ByteBuffer.wrap(buf)); + com.tdunning.math.stats.MergingDigest other = + com.tdunning.math.stats.MergingDigest.fromBytes(java.nio.ByteBuffer.wrap(buf)); tdigest.add(other); } } catch (Exception e) { @@ -1120,7 +1120,7 @@ public class TDigestFunctions { @Override public void output() { if (work.obj != null) { - com.clearspring.analytics.stream.quantile.TDigest tdigest = (com.clearspring.analytics.stream.quantile.TDigest) work.obj; + com.tdunning.math.stats.MergingDigest tdigest = (com.tdunning.math.stats.MergingDigest) work.obj; try { int size = tdigest.smallByteSize(); java.nio.ByteBuffer byteBuf = java.nio.ByteBuffer.allocate(size); @@ -1140,7 +1140,7 @@ public class TDigestFunctions { @Override public void reset() { - work.obj = new com.clearspring.analytics.stream.quantile.TDigest(compression.value); + work.obj = new com.tdunning.math.stats.MergingDigest(compression.value); } } } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/statistics/TDigestMergedStatistic.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/statistics/TDigestMergedStatistic.java index cccbff6..dc84eba 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/statistics/TDigestMergedStatistic.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/statistics/TDigestMergedStatistic.java @@ -20,7 +20,7 @@ package org.apache.drill.exec.physical.impl.statistics; // Library implementing TDigest algorithm to derive approximate quantiles. Please refer to: // 'Computing Extremely Accurate Quantiles using t-Digests' by Ted Dunning and Otmar Ertl -import com.clearspring.analytics.stream.quantile.TDigest; +import com.tdunning.math.stats.MergingDigest; import org.apache.drill.common.types.TypeProtos; import org.apache.drill.exec.ExecConstants; import org.apache.drill.exec.server.options.OptionManager; @@ -33,7 +33,7 @@ import java.util.Map; import java.nio.ByteBuffer; public class TDigestMergedStatistic extends AbstractMergedStatistic { - private Map<String, TDigest> tdigestHolder; + private Map<String, MergingDigest> tdigestHolder; private int compression; public TDigestMergedStatistic() { @@ -63,7 +63,7 @@ public class TDigestMergedStatistic extends AbstractMergedStatistic { assert (input.getField().getType().getMinorType() == TypeProtos.MinorType.MAP); for (ValueVector vv : input) { String colName = vv.getField().getName(); - TDigest colTdigestHolder = null; + MergingDigest colTdigestHolder = null; if (tdigestHolder.get(colName) != null) { colTdigestHolder = tdigestHolder.get(colName); } @@ -71,7 +71,7 @@ public class TDigestMergedStatistic extends AbstractMergedStatistic { NullableVarBinaryVector.Accessor accessor = tdigestVector.getAccessor(); if (!accessor.isNull(0)) { - TDigest other = TDigest.fromBytes(ByteBuffer.wrap(accessor.get(0))); + MergingDigest other = MergingDigest.fromBytes(ByteBuffer.wrap(accessor.get(0))); if (colTdigestHolder != null) { colTdigestHolder.add(other); tdigestHolder.put(colName, colTdigestHolder); @@ -82,7 +82,7 @@ public class TDigestMergedStatistic extends AbstractMergedStatistic { } } - public TDigest getStat(String colName) { + public MergingDigest getStat(String colName) { if (state != State.COMPLETE) { throw new IllegalStateException(String.format("Statistic `%s` has not completed merging statistics", name)); @@ -98,7 +98,7 @@ public class TDigestMergedStatistic extends AbstractMergedStatistic { assert (state == State.MERGE); for (ValueVector outMapCol : output) { String colName = outMapCol.getField().getName(); - TDigest colTdigestHolder = tdigestHolder.get(colName); + MergingDigest colTdigestHolder = tdigestHolder.get(colName); NullableVarBinaryVector vv = (NullableVarBinaryVector) outMapCol; vv.allocateNewSafe(); try { diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/NumericEquiDepthHistogram.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/NumericEquiDepthHistogram.java index 9d5bf6f..0cda7c3 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/NumericEquiDepthHistogram.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/NumericEquiDepthHistogram.java @@ -26,7 +26,7 @@ import java.util.List; import org.apache.calcite.rex.RexCall; import org.apache.calcite.rex.RexNode; import org.apache.calcite.rex.RexLiteral; -import com.clearspring.analytics.stream.quantile.TDigest; +import com.tdunning.math.stats.MergingDigest; import org.apache.calcite.sql.SqlKind; import org.apache.calcite.sql.SqlOperator; import org.apache.drill.shaded.guava.com.google.common.base.Preconditions; @@ -255,7 +255,7 @@ public class NumericEquiDepthHistogram implements Histogram { public static NumericEquiDepthHistogram buildFromTDigest(final byte[] tdigest_array, final int numBuckets, final long nonNullCount) { - TDigest tdigest = TDigest.fromBytes(java.nio.ByteBuffer.wrap(tdigest_array)); + MergingDigest tdigest = MergingDigest.fromBytes(java.nio.ByteBuffer.wrap(tdigest_array)); NumericEquiDepthHistogram histogram = new NumericEquiDepthHistogram(numBuckets);
