TAJO-874: Sometimes InvalidOperationException occurs when aggregates TableStat. (Hyoungjun Kim via hyunsik)
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/4165f339 Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/4165f339 Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/4165f339 Branch: refs/heads/window_function Commit: 4165f33984368c3f0ff504ff65a3f3c0d6904712 Parents: 26265c0 Author: Hyunsik Choi <[email protected]> Authored: Wed Jun 25 17:58:29 2014 +0900 Committer: Hyunsik Choi <[email protected]> Committed: Wed Jun 25 17:58:29 2014 +0900 ---------------------------------------------------------------------- CHANGES | 3 ++ .../tajo/catalog/statistics/StatisticsUtil.java | 44 ++++++++++++-------- .../java/org/apache/tajo/datum/CharDatum.java | 4 +- .../java/org/apache/tajo/datum/DateDatum.java | 2 +- .../java/org/apache/tajo/datum/Float4Datum.java | 6 +-- .../java/org/apache/tajo/datum/Float8Datum.java | 6 +-- .../java/org/apache/tajo/datum/Int2Datum.java | 4 +- .../java/org/apache/tajo/datum/Int4Datum.java | 6 +-- .../java/org/apache/tajo/datum/Int8Datum.java | 4 +- .../org/apache/tajo/datum/IntervalDatum.java | 3 +- .../java/org/apache/tajo/datum/TextDatum.java | 2 +- .../java/org/apache/tajo/datum/TimeDatum.java | 4 +- .../org/apache/tajo/datum/TimestampDatum.java | 4 +- .../apache/tajo/storage/TableStatistics.java | 17 +++++++- 14 files changed, 66 insertions(+), 43 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/4165f339/CHANGES ---------------------------------------------------------------------- diff --git a/CHANGES b/CHANGES index c10dbdf..a80986c 100644 --- a/CHANGES +++ b/CHANGES @@ -74,6 +74,9 @@ Release 0.9.0 - unreleased BUG FIXES + TAJO-874: Sometimes InvalidOperationException occurs when aggregates + TableStat. (Hyoungjun Kim via hyunsik) + TAJO-866: COUNT DISTINCT with other aggregation function throws ClassCastException. (Hyoungjun Kim via hyunsik) http://git-wip-us.apache.org/repos/asf/tajo/blob/4165f339/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/statistics/StatisticsUtil.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/statistics/StatisticsUtil.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/statistics/StatisticsUtil.java index 01316bc..c481276 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/statistics/StatisticsUtil.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/statistics/StatisticsUtil.java @@ -65,15 +65,19 @@ public class StatisticsUtil { continue; } - agg.setNumDistVals(agg.getNumDistValues() + cs.getNumDistValues()); - agg.setNumNulls(agg.getNumNulls() + cs.getNumNulls()); - if (!cs.minIsNotSet() && (agg.minIsNotSet() || - agg.getMinValue().compareTo(cs.getMinValue()) > 0)) { - agg.setMinValue(cs.getMinValue()); - } - if (!cs.maxIsNotSet() && (agg.maxIsNotSet() || - agg.getMaxValue().compareTo(cs.getMaxValue()) < 0)) { - agg.setMaxValue(stats.getColumnStats().get(i).getMaxValue()); + try { + agg.setNumDistVals(agg.getNumDistValues() + cs.getNumDistValues()); + agg.setNumNulls(agg.getNumNulls() + cs.getNumNulls()); + if (!cs.minIsNotSet() && (agg.minIsNotSet() || + agg.getMinValue().compareTo(cs.getMinValue()) > 0)) { + agg.setMinValue(cs.getMinValue()); + } + if (!cs.maxIsNotSet() && (agg.maxIsNotSet() || + agg.getMaxValue().compareTo(cs.getMaxValue()) < 0)) { + agg.setMaxValue(stats.getColumnStats().get(i).getMaxValue()); + } + } catch (Exception e) { + LOG.warn(e.getMessage(), e); } } } @@ -117,15 +121,19 @@ public class StatisticsUtil { LOG.warn("ERROR: One of column stats is NULL (expected column: " + css[i].getColumn() + ")"); continue; } - css[i].setNumDistVals(css[i].getNumDistValues() + cs.getNumDistValues()); - css[i].setNumNulls(css[i].getNumNulls() + cs.getNumNulls()); - if (!cs.minIsNotSet() && (css[i].minIsNotSet() || - css[i].getMinValue().compareTo(cs.getMinValue()) > 0)) { - css[i].setMinValue(cs.getMinValue()); - } - if (!cs.maxIsNotSet() && (css[i].maxIsNotSet() || - css[i].getMaxValue().compareTo(cs.getMaxValue()) < 0)) { - css[i].setMaxValue(ts.getColumnStats().get(i).getMaxValue()); + try { + css[i].setNumDistVals(css[i].getNumDistValues() + cs.getNumDistValues()); + css[i].setNumNulls(css[i].getNumNulls() + cs.getNumNulls()); + if (!cs.minIsNotSet() && (css[i].minIsNotSet() || + css[i].getMinValue().compareTo(cs.getMinValue()) > 0)) { + css[i].setMinValue(cs.getMinValue()); + } + if (!cs.maxIsNotSet() && (css[i].maxIsNotSet() || + css[i].getMaxValue().compareTo(cs.getMaxValue()) < 0)) { + css[i].setMaxValue(ts.getColumnStats().get(i).getMaxValue()); + } + } catch (Exception e) { + LOG.warn(e.getMessage(), e); } } } http://git-wip-us.apache.org/repos/asf/tajo/blob/4165f339/tajo-common/src/main/java/org/apache/tajo/datum/CharDatum.java ---------------------------------------------------------------------- diff --git a/tajo-common/src/main/java/org/apache/tajo/datum/CharDatum.java b/tajo-common/src/main/java/org/apache/tajo/datum/CharDatum.java index 444ca66..e6c4d94 100644 --- a/tajo-common/src/main/java/org/apache/tajo/datum/CharDatum.java +++ b/tajo-common/src/main/java/org/apache/tajo/datum/CharDatum.java @@ -141,7 +141,7 @@ public class CharDatum extends Datum { return datum; default: - throw new InvalidOperationException(); + throw new InvalidOperationException(datum.type()); } } @@ -156,7 +156,7 @@ public class CharDatum extends Datum { return -1; default: - throw new InvalidOperationException(); + throw new InvalidOperationException(datum.type()); } } } http://git-wip-us.apache.org/repos/asf/tajo/blob/4165f339/tajo-common/src/main/java/org/apache/tajo/datum/DateDatum.java ---------------------------------------------------------------------- diff --git a/tajo-common/src/main/java/org/apache/tajo/datum/DateDatum.java b/tajo-common/src/main/java/org/apache/tajo/datum/DateDatum.java index 10f7077..b26ef84 100644 --- a/tajo-common/src/main/java/org/apache/tajo/datum/DateDatum.java +++ b/tajo-common/src/main/java/org/apache/tajo/datum/DateDatum.java @@ -254,7 +254,7 @@ public class DateDatum extends Datum { } else if (datum instanceof NullDatum || datum.isNull()) { return -1; } else { - throw new InvalidOperationException(); + throw new InvalidOperationException(datum.type()); } } http://git-wip-us.apache.org/repos/asf/tajo/blob/4165f339/tajo-common/src/main/java/org/apache/tajo/datum/Float4Datum.java ---------------------------------------------------------------------- diff --git a/tajo-common/src/main/java/org/apache/tajo/datum/Float4Datum.java b/tajo-common/src/main/java/org/apache/tajo/datum/Float4Datum.java index c269a40..e24bce4 100644 --- a/tajo-common/src/main/java/org/apache/tajo/datum/Float4Datum.java +++ b/tajo-common/src/main/java/org/apache/tajo/datum/Float4Datum.java @@ -135,7 +135,7 @@ public class Float4Datum extends NumericDatum { case NULL_TYPE: return datum; default: - throw new InvalidOperationException(); + throw new InvalidOperationException(datum.type()); } } @@ -195,7 +195,7 @@ public class Float4Datum extends NumericDatum { case NULL_TYPE: return -1; default: - throw new InvalidOperationException(); + throw new InvalidOperationException(datum.type()); } } @@ -268,7 +268,7 @@ public class Float4Datum extends NumericDatum { case NULL_TYPE: return datum; default: - throw new InvalidOperationException(); + throw new InvalidOperationException(datum.type()); } } http://git-wip-us.apache.org/repos/asf/tajo/blob/4165f339/tajo-common/src/main/java/org/apache/tajo/datum/Float8Datum.java ---------------------------------------------------------------------- diff --git a/tajo-common/src/main/java/org/apache/tajo/datum/Float8Datum.java b/tajo-common/src/main/java/org/apache/tajo/datum/Float8Datum.java index b4cfc67..0542148 100644 --- a/tajo-common/src/main/java/org/apache/tajo/datum/Float8Datum.java +++ b/tajo-common/src/main/java/org/apache/tajo/datum/Float8Datum.java @@ -124,7 +124,7 @@ public class Float8Datum extends NumericDatum { case NULL_TYPE: return datum; default: - throw new InvalidOperationException(); + throw new InvalidOperationException(datum.type()); } } @@ -184,7 +184,7 @@ public class Float8Datum extends NumericDatum { case NULL_TYPE: return -1; default: - throw new InvalidOperationException(); + throw new InvalidOperationException(datum.type()); } } @@ -257,7 +257,7 @@ public class Float8Datum extends NumericDatum { case NULL_TYPE: return datum; default: - throw new InvalidOperationException(); + throw new InvalidOperationException(datum.type()); } } http://git-wip-us.apache.org/repos/asf/tajo/blob/4165f339/tajo-common/src/main/java/org/apache/tajo/datum/Int2Datum.java ---------------------------------------------------------------------- diff --git a/tajo-common/src/main/java/org/apache/tajo/datum/Int2Datum.java b/tajo-common/src/main/java/org/apache/tajo/datum/Int2Datum.java index 7770b94..38cf019 100644 --- a/tajo-common/src/main/java/org/apache/tajo/datum/Int2Datum.java +++ b/tajo-common/src/main/java/org/apache/tajo/datum/Int2Datum.java @@ -126,7 +126,7 @@ public class Int2Datum extends NumericDatum { case NULL_TYPE: return datum; default: - throw new InvalidOperationException(); + throw new InvalidOperationException(datum.type()); } } @@ -186,7 +186,7 @@ public class Int2Datum extends NumericDatum { case NULL_TYPE: return -1; default: - throw new InvalidOperationException(); + throw new InvalidOperationException(datum.type()); } } http://git-wip-us.apache.org/repos/asf/tajo/blob/4165f339/tajo-common/src/main/java/org/apache/tajo/datum/Int4Datum.java ---------------------------------------------------------------------- diff --git a/tajo-common/src/main/java/org/apache/tajo/datum/Int4Datum.java b/tajo-common/src/main/java/org/apache/tajo/datum/Int4Datum.java index 7f7ac13..d26b6b2 100644 --- a/tajo-common/src/main/java/org/apache/tajo/datum/Int4Datum.java +++ b/tajo-common/src/main/java/org/apache/tajo/datum/Int4Datum.java @@ -130,7 +130,7 @@ public class Int4Datum extends NumericDatum { case NULL_TYPE: return datum; default: - throw new InvalidOperationException(); + throw new InvalidOperationException(datum.type()); } } @@ -190,7 +190,7 @@ public class Int4Datum extends NumericDatum { case NULL_TYPE: return -1; default: - throw new InvalidOperationException(); + throw new InvalidOperationException(datum.type()); } } @@ -263,7 +263,7 @@ public class Int4Datum extends NumericDatum { case NULL_TYPE: return datum; default: - throw new InvalidOperationException(); + throw new InvalidOperationException(datum.type()); } } http://git-wip-us.apache.org/repos/asf/tajo/blob/4165f339/tajo-common/src/main/java/org/apache/tajo/datum/Int8Datum.java ---------------------------------------------------------------------- diff --git a/tajo-common/src/main/java/org/apache/tajo/datum/Int8Datum.java b/tajo-common/src/main/java/org/apache/tajo/datum/Int8Datum.java index 3b1e67d..46a1353 100644 --- a/tajo-common/src/main/java/org/apache/tajo/datum/Int8Datum.java +++ b/tajo-common/src/main/java/org/apache/tajo/datum/Int8Datum.java @@ -137,7 +137,7 @@ public class Int8Datum extends NumericDatum { case NULL_TYPE: return datum; default: - throw new InvalidOperationException(); + throw new InvalidOperationException(datum.type()); } } @@ -198,7 +198,7 @@ public class Int8Datum extends NumericDatum { return -1; default: - throw new InvalidOperationException(); + throw new InvalidOperationException(datum.type()); } } http://git-wip-us.apache.org/repos/asf/tajo/blob/4165f339/tajo-common/src/main/java/org/apache/tajo/datum/IntervalDatum.java ---------------------------------------------------------------------- diff --git a/tajo-common/src/main/java/org/apache/tajo/datum/IntervalDatum.java b/tajo-common/src/main/java/org/apache/tajo/datum/IntervalDatum.java index 5ad9635..28a0a47 100644 --- a/tajo-common/src/main/java/org/apache/tajo/datum/IntervalDatum.java +++ b/tajo-common/src/main/java/org/apache/tajo/datum/IntervalDatum.java @@ -96,7 +96,6 @@ public class IntervalDatum extends Datum { int length = intervalStr.getBytes().length; - int start = 0; StringBuilder digitChars = new StringBuilder(); StringBuilder unitChars = new StringBuilder(); for (int i = 0; i < length; i++) { @@ -406,7 +405,7 @@ public class IntervalDatum extends Datum { } else if (datum instanceof NullDatum || datum.isNull()) { return -1; } else { - throw new InvalidOperationException(); + throw new InvalidOperationException(datum.type()); } } http://git-wip-us.apache.org/repos/asf/tajo/blob/4165f339/tajo-common/src/main/java/org/apache/tajo/datum/TextDatum.java ---------------------------------------------------------------------- diff --git a/tajo-common/src/main/java/org/apache/tajo/datum/TextDatum.java b/tajo-common/src/main/java/org/apache/tajo/datum/TextDatum.java index 313b905..49f09f6 100644 --- a/tajo-common/src/main/java/org/apache/tajo/datum/TextDatum.java +++ b/tajo-common/src/main/java/org/apache/tajo/datum/TextDatum.java @@ -110,7 +110,7 @@ public class TextDatum extends Datum { return -1; default: - throw new InvalidOperationException(); + throw new InvalidOperationException(datum.type()); } } http://git-wip-us.apache.org/repos/asf/tajo/blob/4165f339/tajo-common/src/main/java/org/apache/tajo/datum/TimeDatum.java ---------------------------------------------------------------------- diff --git a/tajo-common/src/main/java/org/apache/tajo/datum/TimeDatum.java b/tajo-common/src/main/java/org/apache/tajo/datum/TimeDatum.java index a7b9d1e..37e5e78 100644 --- a/tajo-common/src/main/java/org/apache/tajo/datum/TimeDatum.java +++ b/tajo-common/src/main/java/org/apache/tajo/datum/TimeDatum.java @@ -167,7 +167,7 @@ public class TimeDatum extends Datum { } else if (datum.isNull()) { return datum; } else { - throw new InvalidOperationException(); + throw new InvalidOperationException(datum.type()); } } @@ -179,7 +179,7 @@ public class TimeDatum extends Datum { } else if (datum instanceof NullDatum || datum.isNull()) { return -1; } else { - throw new InvalidOperationException(); + throw new InvalidOperationException(datum.type()); } } http://git-wip-us.apache.org/repos/asf/tajo/blob/4165f339/tajo-common/src/main/java/org/apache/tajo/datum/TimestampDatum.java ---------------------------------------------------------------------- diff --git a/tajo-common/src/main/java/org/apache/tajo/datum/TimestampDatum.java b/tajo-common/src/main/java/org/apache/tajo/datum/TimestampDatum.java index d3cc5f4..62b0861 100644 --- a/tajo-common/src/main/java/org/apache/tajo/datum/TimestampDatum.java +++ b/tajo-common/src/main/java/org/apache/tajo/datum/TimestampDatum.java @@ -170,7 +170,7 @@ public class TimestampDatum extends Datum { } else if (datum.isNull()) { return datum; } else { - throw new RuntimeException(); + throw new InvalidOperationException(datum.type()); } } @@ -182,7 +182,7 @@ public class TimestampDatum extends Datum { } else if (datum.isNull()) { return -1; } else { - throw new RuntimeException(); + throw new InvalidOperationException(datum.type()); } } http://git-wip-us.apache.org/repos/asf/tajo/blob/4165f339/tajo-storage/src/main/java/org/apache/tajo/storage/TableStatistics.java ---------------------------------------------------------------------- diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/TableStatistics.java b/tajo-storage/src/main/java/org/apache/tajo/storage/TableStatistics.java index cbee95e..ac9bd8a 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/TableStatistics.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/TableStatistics.java @@ -18,6 +18,8 @@ package org.apache.tajo.storage; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.tajo.catalog.Schema; import org.apache.tajo.catalog.statistics.ColumnStats; import org.apache.tajo.catalog.statistics.TableStats; @@ -30,6 +32,7 @@ import org.apache.tajo.datum.NullDatum; * This class is not thread-safe. */ public class TableStatistics { + private static final Log LOG = LogFactory.getLog(TableStatistics.class); private Schema schema; private Tuple minValues; private Tuple maxValues; @@ -104,8 +107,18 @@ public class TableStatistics { for (int i = 0; i < schema.size(); i++) { columnStats = new ColumnStats(schema.getColumn(i)); columnStats.setNumNulls(numNulls[i]); - columnStats.setMinValue(minValues.get(i)); - columnStats.setMaxValue(maxValues.get(i)); + if (minValues.get(i) == null || schema.getColumn(i).getDataType().getType() == minValues.get(i).type()) { + columnStats.setMinValue(minValues.get(i)); + } else { + LOG.warn("Wrong statistics column type (" + minValues.get(i).type() + + ", expected=" + schema.getColumn(i).getDataType().getType() + ")"); + } + if (minValues.get(i) == null || schema.getColumn(i).getDataType().getType() == minValues.get(i).type()) { + columnStats.setMaxValue(maxValues.get(i)); + } else { + LOG.warn("Wrong statistics column type (" + minValues.get(i).type() + + ", expected=" + schema.getColumn(i).getDataType().getType() + ")"); + } stat.addColumnStat(columnStats); }
