ilooner closed pull request #1406: DRILL-6641: Fix columnValueCounts in
ParquetGroupScanStatistics when ParquetGroupScan has RowGroupInfo without
column statistics
URL: https://github.com/apache/drill/pull/1406
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetGroupScanStatistics.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetGroupScanStatistics.java
index f7d56871951..9381043b575 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetGroupScanStatistics.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetGroupScanStatistics.java
@@ -17,6 +17,7 @@
*/
package org.apache.drill.exec.store.parquet;
+import org.apache.commons.lang3.mutable.MutableLong;
import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.common.types.TypeProtos;
import org.apache.drill.exec.physical.base.GroupScan;
@@ -44,7 +45,7 @@
// only for partition columns : value is unique for each partition
private Map<SchemaPath, TypeProtos.MajorType> partitionColTypeMap;
// total number of non-null value for each column in parquet files
- private Map<SchemaPath, Long> columnValueCounts;
+ private Map<SchemaPath, MutableLong> columnValueCounts;
// total number of rows (obtained from parquet footer)
private long rowCount;
@@ -61,7 +62,8 @@ public ParquetGroupScanStatistics(ParquetGroupScanStatistics
that) {
}
public long getColumnValueCount(SchemaPath column) {
- return columnValueCounts.containsKey(column) ?
columnValueCounts.get(column) : 0;
+ MutableLong count = columnValueCounts.get(column);
+ return count != null ? count.getValue() : 0;
}
public List<SchemaPath> getPartitionColumns() {
@@ -87,19 +89,15 @@ public void collect(List<RowGroupInfo> rowGroupInfos,
ParquetTableMetadataBase p
long rowCount = rowGroup.getRowCount();
for (ColumnMetadata column : rowGroup.getColumns()) {
SchemaPath schemaPath = SchemaPath.getCompoundPath(column.getName());
- Long previousCount = columnValueCounts.get(schemaPath);
- if (previousCount != null) {
- if (previousCount != GroupScan.NO_COLUMN_STATS &&
column.isNumNullsSet()) {
- Long newCount = rowCount - column.getNulls();
- columnValueCounts.put(schemaPath,
columnValueCounts.get(schemaPath) + newCount);
- }
+ MutableLong emptyCount = new MutableLong();
+ MutableLong previousCount = columnValueCounts.putIfAbsent(schemaPath,
emptyCount);
+ if (previousCount == null) {
+ previousCount = emptyCount;
+ }
+ if (previousCount.longValue() != GroupScan.NO_COLUMN_STATS &&
column.isNumNullsSet()) {
+ previousCount.add(rowCount - column.getNulls());
} else {
- if (column.isNumNullsSet()) {
- Long newCount = rowCount - column.getNulls();
- columnValueCounts.put(schemaPath, newCount);
- } else {
- columnValueCounts.put(schemaPath, GroupScan.NO_COLUMN_STATS);
- }
+ previousCount.setValue(GroupScan.NO_COLUMN_STATS);
}
boolean partitionColumn = checkForPartitionColumn(column, first,
rowCount, parquetTableMetadata);
if (partitionColumn) {
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
