This is an automated email from the ASF dual-hosted git repository.
deniskuzZ pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 2694126f934 HIVE-30106: Iceberg: Column-stats ANALYZE spends most of
its CPU on per-row reader plumbing (#6834)
2694126f934 is described below
commit 2694126f934635e12310ea3b9b483de05e2fa7b2
Author: Denys Kuzmenko <[email protected]>
AuthorDate: Thu Oct 22 17:51:41 2026 +0200
HIVE-30106: Iceberg: Column-stats ANALYZE spends most of its CPU on per-row
reader plumbing (#6834)
---
.../iceberg/mr/hive/HiveIcebergStorageHandler.java | 46 ++++++--
.../apache/iceberg/mr/hive/IcebergAcidUtil.java | 34 +++---
.../apache/iceberg/mr/hive/IcebergTableUtil.java | 22 ++--
.../mr/mapred/MapredIcebergInputFormat.java | 34 +++---
.../iceberg/mr/mapreduce/IcebergRecordReader.java | 17 ++-
.../iceberg/mr/hive/TestHiveIcebergCRUD.java | 9 +-
.../iceberg/mr/hive/TestHiveIcebergTimeTravel.java | 29 +++++
.../positive/iceberg_row_mode_virtual_columns.q | 20 ++++
.../src/test/queries/positive/iceberg_stats.q | 28 +++++
.../positive/vectorized_iceberg_read_parquet.q | 2 +
.../src/test/results/positive/iceberg_stats.q.out | 126 +++++++++++++++++++++
.../llap/iceberg_row_mode_virtual_columns.q.out | 87 ++++++++++++++
.../llap/vectorized_iceberg_read_parquet.q.out | 47 ++++++++
.../positive/vectorized_iceberg_read_parquet.q.out | 46 ++++++++
.../test/resources/testconfiguration.properties | 2 +
.../org/apache/hadoop/hive/ql/exec/Utilities.java | 8 ++
.../hive/ql/io/HiveContextAwareRecordReader.java | 5 +-
.../org/apache/hadoop/hive/ql/io/IOContext.java | 9 +-
.../hive/ql/io/IOContextAwareRecordReader.java | 33 ++++++
.../hadoop/hive/ql/io/PositionDeleteInfo.java | 22 ----
.../apache/hadoop/hive/ql/io/RowLineageInfo.java | 25 ----
.../hive/ql/metadata/HiveStorageHandler.java | 15 ++-
.../hadoop/hive/ql/parse/SemanticAnalyzer.java | 63 ++++++-----
.../apache/hadoop/hive/ql/parse/TaskCompiler.java | 6 +-
.../hadoop/hive/ql/stats/BasicStatsNoJobTask.java | 61 +++++-----
.../hadoop/hive/ql/stats/BasicStatsTask.java | 4 +-
26 files changed, 626 insertions(+), 174 deletions(-)
diff --git
a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java
index 807ed4ee057..9295bdfc31f 100644
---
a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java
+++
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java
@@ -198,6 +198,7 @@
import org.apache.iceberg.relocated.com.google.common.collect.Maps;
import org.apache.iceberg.relocated.com.google.common.collect.Sets;
import org.apache.iceberg.types.Types;
+import org.apache.iceberg.util.PropertyUtil;
import org.apache.iceberg.util.SerializationUtil;
import org.apache.iceberg.util.SnapshotUtil;
import org.slf4j.Logger;
@@ -294,8 +295,9 @@ public void configureInputJobCredentials(TableDesc
tableDesc, Map<String, String
@Override
public void configureInputJobProperties(TableDesc tableDesc, Map<String,
String> map) {
overlayTableProperties(conf, tableDesc, map);
- // Until the vectorized reader can handle delete files, let's fall back to
non-vector mode for V2 tables
- fallbackToNonVectorizedModeBasedOnProperties(tableDesc.getProperties());
+ if (vectorizationUnsupported(tableDesc.getProperties()) ||
hasEqualityDeletes(tableDesc.getProperties())) {
+ disableVectorization();
+ }
boolean allowDataFilesWithinTableLocationOnly =
conf.getBoolean(ConfVars.HIVE_ICEBERG_ALLOW_DATAFILES_IN_TABLE_LOCATION_ONLY.varname,
@@ -308,8 +310,9 @@ public void configureInputJobProperties(TableDesc
tableDesc, Map<String, String>
@Override
public void configureOutputJobProperties(TableDesc tableDesc, Map<String,
String> map) {
overlayTableProperties(conf, tableDesc, map);
- // Until the vectorized reader can handle delete files, let's fall back to
non-vector mode for V2 tables
- fallbackToNonVectorizedModeBasedOnProperties(tableDesc.getProperties());
+ if (vectorizationUnsupported(tableDesc.getProperties())) {
+ disableVectorization();
+ }
// For Tez, setting the committer here is enough to make sure it'll be
part of the jobConf
map.put("mapred.output.committer.class",
HiveIcebergNoJobCommitter.class.getName());
// For MR, the jobConf is set only in configureJobConf, so we're setting
the write key here to detect it over there
@@ -443,6 +446,11 @@ public boolean canProvideBasicStatistics() {
return true;
}
+ @Override
+ public boolean
canProvideBasicStatistics(org.apache.hadoop.hive.ql.metadata.Table hmsTable) {
+ return canProvideBasicStatistics() &&
HiveMetaHook.ICEBERG.equals(getStatsSource());
+ }
+
@Override
public boolean
canProvidePartitionStatistics(org.apache.hadoop.hive.ql.metadata.Table
hmsTable) {
if (!getStatsSource().equals(HiveMetaHook.ICEBERG)) {
@@ -1872,7 +1880,7 @@ private String
collectColumnAndReplaceDummyValues(ExprNodeDesc node, String foun
}
/**
- * If any of the following checks is true we fall back to non vectorized
mode:
+ * Whether the table cannot be read vectorized:
* <ul>
* <li>fileformat is set to avro</li>
* <li>querying metadata tables</li>
@@ -1881,17 +1889,31 @@ private String
collectColumnAndReplaceDummyValues(ExprNodeDesc node, String foun
* </ul>
* @param tableProps table properties, must be not null
*/
- private void fallbackToNonVectorizedModeBasedOnProperties(Properties
tableProps) {
+ private boolean vectorizationUnsupported(Properties tableProps) {
Schema tableSchema =
SchemaParser.fromJson(tableProps.getProperty(InputFormatConfig.TABLE_SCHEMA));
- if (FileFormat.AVRO ==
IcebergTableUtil.defaultFileFormat(tableProps::getProperty) ||
+ return FileFormat.AVRO ==
IcebergTableUtil.defaultFileFormat(tableProps::getProperty) ||
isValidMetadataTable(tableProps.getProperty(IcebergAcidUtil.META_TABLE_PROPERTY))
||
hasOrcTimeInSchema(tableProps, tableSchema) ||
- !hasParquetNestedTypeWithinListOrMap(tableProps, tableSchema)) {
- // disable vectorization
- SessionStateUtil.getQueryState(conf).ifPresent(qs ->
- qs.getConf().setBoolVar(ConfVars.HIVE_VECTORIZATION_ENABLED, false));
- }
+ !hasParquetNestedTypeWithinListOrMap(tableProps, tableSchema);
+ }
+
+ /**
+ * Whether the scanned snapshot has equality deletes, which the vectorized
reader cannot apply
+ * (HiveVectorizedReader): the one read as of a time or version, the head of
the branch or tag, or the current one.
+ */
+ private boolean hasEqualityDeletes(Properties tableProps) {
+ Table table = IcebergTableUtil.getTable(conf, tableProps);
+ Snapshot snapshot = IcebergTableUtil.getTableSnapshot(table,
tableProps.getProperty(Utilities.AS_OF_TIMESTAMP),
+ tableProps.getProperty(Utilities.AS_OF_VERSION),
tableProps.getProperty(Catalogs.SNAPSHOT_REF));
+ // a V1 snapshot may have no summary, and V1 has no delete files
+ return snapshot != null && snapshot.summary() != null &&
+ PropertyUtil.propertyAsLong(snapshot.summary(), TOTAL_EQ_DELETES_PROP,
0) > 0;
+ }
+
+ private void disableVectorization() {
+ SessionStateUtil.getQueryState(conf).ifPresent(qs ->
+ qs.getConf().setBoolVar(ConfVars.HIVE_VECTORIZATION_ENABLED, false));
}
/**
diff --git
a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergAcidUtil.java
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergAcidUtil.java
index a28423cdd9f..38672064163 100644
---
a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergAcidUtil.java
+++
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergAcidUtil.java
@@ -24,9 +24,10 @@
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
+import java.util.function.Supplier;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.ql.io.IOContextMap;
+import org.apache.hadoop.hive.ql.io.IOContext;
import org.apache.hadoop.hive.ql.io.PositionDeleteInfo;
import org.apache.hadoop.hive.ql.io.RowLineageInfo;
import org.apache.hadoop.hive.ql.lockmgr.HiveTxnManager;
@@ -181,10 +182,6 @@ public static String getFilePath(Record rec) {
return rec.get(DELETE_FILE_META_COLS.get(MetadataColumns.FILE_PATH),
String.class);
}
- public static long getFilePosition(Record rec) {
- return rec.get(FILE_READ_META_COLS.get(MetadataColumns.ROW_POSITION),
Long.class);
- }
-
public static long getDeleteFilePosition(Record rec) {
return rec.get(DELETE_FILE_META_COLS.get(MetadataColumns.ROW_POSITION),
Long.class);
}
@@ -240,27 +237,29 @@ public static Transaction getTransaction(Table table) {
public static class VirtualColumnAwareIterator<T> implements
CloseableIterator<T> {
+ private static final int ROW_POSITION_INDEX =
FILE_READ_META_COLS.get(MetadataColumns.ROW_POSITION);
+
private final CloseableIterator<T> currentIterator;
private final GenericRecord current;
- private final Configuration conf;
+ // the reader's IOContext, read per row: the reader opens its first file
before the wrapper hands it over
+ private final Supplier<IOContext> ioContext;
private final int specId;
private final long partitionHash;
private final String filePath;
+ private final String partitionName;
public VirtualColumnAwareIterator(CloseableIterator<T> currentIterator,
List<Types.NestedField> columns,
- Configuration conf, FileScanTask task) {
+ FileScanTask task, Supplier<IOContext> ioContext) {
this.currentIterator = currentIterator;
this.current = GenericRecord.create(
new Schema(columns.subList(FILE_READ_META_COLS.size(),
columns.size())));
- this.conf = conf;
+ this.ioContext = ioContext;
this.specId = task.file().specId();
this.partitionHash = computeHash(task.file().partition());
this.filePath = task.file().location();
-
- IOContextMap.get(conf).setPartitionName(
- IcebergTableUtil.toPartitionName(task.spec(),
task.file().partition()));
+ this.partitionName = IcebergTableUtil.toPartitionName(task.spec(),
task.file().partition());
}
@Override
@@ -278,13 +277,12 @@ public T next() {
T next = currentIterator.next();
GenericRecord rec = (GenericRecord) next;
IcebergAcidUtil.copyFields(rec, FILE_READ_META_COLS.size(),
current.size(), current);
- PositionDeleteInfo.setIntoConf(conf,
- specId,
- partitionHash,
- filePath,
- IcebergAcidUtil.getFilePosition(rec));
- RowLineageInfo.setRowLineageInfoIntoConf(RowLineageReader.readRowId(rec),
- RowLineageReader.readLastUpdatedSequenceNumber(rec), conf);
+ IOContext ioCxt = ioContext.get();
+ ioCxt.setPartitionName(partitionName);
+ ioCxt.setPositionDeleteInfo(new PositionDeleteInfo(
+ specId, partitionHash, filePath, rec.get(ROW_POSITION_INDEX,
Long.class)));
+ ioCxt.setRowLineageInfo(new RowLineageInfo(
+ RowLineageReader.readRowId(rec),
RowLineageReader.readLastUpdatedSequenceNumber(rec)));
return (T) current;
}
}
diff --git
a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergTableUtil.java
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergTableUtil.java
index 666aeda3a5a..eec354ec8e9 100644
---
a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergTableUtil.java
+++
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergTableUtil.java
@@ -214,22 +214,30 @@ static Table getTable(Configuration configuration,
Properties properties) {
}
public static Snapshot getTableSnapshot(Table table,
org.apache.hadoop.hive.ql.metadata.Table hmsTable) {
+ return getTableSnapshot(table, hmsTable.getAsOfTimestamp(),
hmsTable.getAsOfVersion(), hmsTable.getSnapshotRef());
+ }
+
+ /**
+ * The snapshot a scan reads: as of a time, as of a version (a snapshot id
or a ref name), the head of a branch or
+ * tag, or the current one.
+ */
+ static Snapshot getTableSnapshot(Table table, String asOfTimestamp, String
asOfVersion, String snapshotRef) {
long snapshotId = -1;
- if (hmsTable.getAsOfTimestamp() != null) {
+ if (asOfTimestamp != null) {
ZoneId timeZone = SessionState.get() == null ?
new HiveConf().getLocalTimeZone() :
SessionState.get().getConf().getLocalTimeZone();
- TimestampTZ time = TimestampTZUtil.parse(hmsTable.getAsOfTimestamp(),
timeZone);
+ TimestampTZ time = TimestampTZUtil.parse(asOfTimestamp, timeZone);
snapshotId = SnapshotUtil.snapshotIdAsOfTime(table, time.toEpochMilli());
- } else if (hmsTable.getAsOfVersion() != null) {
+ } else if (asOfVersion != null) {
try {
- snapshotId = Long.parseLong(hmsTable.getAsOfVersion());
+ snapshotId = Long.parseLong(asOfVersion);
} catch (NumberFormatException e) {
- SnapshotRef ref = table.refs().get(hmsTable.getAsOfVersion());
+ SnapshotRef ref = table.refs().get(asOfVersion);
if (ref == null) {
throw new RuntimeException("Cannot find matching snapshot ID or
reference name for version " +
- hmsTable.getAsOfVersion());
+ asOfVersion);
}
snapshotId = ref.snapshotId();
}
@@ -237,7 +245,7 @@ public static Snapshot getTableSnapshot(Table table,
org.apache.hadoop.hive.ql.m
if (snapshotId > 0) {
return table.snapshot(snapshotId);
}
- return getTableSnapshot(table, hmsTable.getSnapshotRef());
+ return getTableSnapshot(table, snapshotRef);
}
static Snapshot getTableSnapshot(Table table, String snapshotRef) {
diff --git
a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/mapred/MapredIcebergInputFormat.java
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/mapred/MapredIcebergInputFormat.java
index 473e8fe1486..578ead100ef 100644
---
a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/mapred/MapredIcebergInputFormat.java
+++
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/mapred/MapredIcebergInputFormat.java
@@ -22,6 +22,8 @@
import java.io.IOException;
import java.util.Optional;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.ql.io.IOContext;
+import org.apache.hadoop.hive.ql.io.IOContextAwareRecordReader;
import org.apache.hadoop.mapred.InputFormat;
import org.apache.hadoop.mapred.InputSplit;
import org.apache.hadoop.mapred.JobConf;
@@ -75,25 +77,18 @@ public InputSplit[] getSplits(JobConf job, int numSplits)
throws IOException {
@Override
public RecordReader<Void, Container<T>> getRecordReader(InputSplit split,
JobConf job,
Reporter reporter)
throws IOException {
- try {
- org.apache.hadoop.mapreduce.InputSplit inputSplit = split instanceof
IcebergMergeSplit ?
- (IcebergMergeSplit) split : ((IcebergSplitContainer)
split).icebergSplit();
- return new MapredIcebergRecordReader<>(innerInputFormat, inputSplit,
job, reporter);
- } catch (InterruptedException e) {
- throw new RuntimeException(e);
- }
+ org.apache.hadoop.mapreduce.InputSplit inputSplit = split instanceof
IcebergMergeSplit ?
+ (IcebergMergeSplit) split : ((IcebergSplitContainer)
split).icebergSplit();
+ return new MapredIcebergRecordReader<>(innerInputFormat, inputSplit, job,
reporter);
}
- private static final class MapredIcebergRecordReader<T> extends
AbstractMapredIcebergRecordReader<Container<T>> {
-
- private final long splitLength; // for getPos()
+ private static final class MapredIcebergRecordReader<T> extends
AbstractMapredIcebergRecordReader<Container<T>>
+ implements IOContextAwareRecordReader {
MapredIcebergRecordReader(org.apache.iceberg.mr.mapreduce.IcebergInputFormat<T>
mapreduceInputFormat,
- org.apache.hadoop.mapreduce.InputSplit split, JobConf job, Reporter
reporter)
- throws IOException, InterruptedException {
+ org.apache.hadoop.mapreduce.InputSplit split, JobConf job, Reporter
reporter) throws IOException {
super(mapreduceInputFormat, split, job, reporter);
- splitLength = split.getLength();
}
@Override
@@ -116,9 +111,20 @@ public Container<T> createValue() {
return new Container<>();
}
+ @Override
+ public void setIOContext(IOContext ioContext) {
+ if (innerReader instanceof IOContextAwareRecordReader reader) {
+ reader.setIOContext(ioContext);
+ }
+ }
+
+ /**
+ * A split is a group of file scan tasks, so no position within it maps to
a file offset. -1 keeps
+ * HiveContextAwareRecordReader from taking every row for the start of a
file.
+ */
@Override
public long getPos() throws IOException {
- return (long) (splitLength * getProgress());
+ return -1;
}
}
diff --git
a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergRecordReader.java
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergRecordReader.java
index 14430fdc3af..2ea261f2014 100644
---
a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergRecordReader.java
+++
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergRecordReader.java
@@ -30,6 +30,8 @@
import java.util.stream.Stream;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.ql.exec.Utilities;
+import org.apache.hadoop.hive.ql.io.IOContext;
+import org.apache.hadoop.hive.ql.io.IOContextAwareRecordReader;
import org.apache.hadoop.mapreduce.InputSplit;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.apache.iceberg.DataFile;
@@ -69,10 +71,11 @@
import org.apache.iceberg.types.Types;
import org.apache.iceberg.util.PartitionUtil;
-public final class IcebergRecordReader<T> extends
AbstractIcebergRecordReader<T> {
+public final class IcebergRecordReader<T> extends
AbstractIcebergRecordReader<T> implements IOContextAwareRecordReader {
private Iterator<FileScanTask> tasks;
private CloseableIterator<T> currentIterator;
private T current;
+ private IOContext ioContext;
@Override
public void initialize(InputSplit split, TaskAttemptContext newContext) {
@@ -83,6 +86,16 @@ public void initialize(InputSplit split, TaskAttemptContext
newContext) {
this.currentIterator = nextTask();
}
+ /**
+ * Sets the IOContext the virtual columns of each row are written into.
MapredIcebergRecordReader passes on the
+ * IOContext its wrapper hands over after initialize() opened the first
file, because HiveInputFormat handles an
+ * interrupt only while it creates the reader (HIVE-25751); each file's
iterator therefore reads it per row.
+ */
+ @Override
+ public void setIOContext(IOContext ioCxt) {
+ this.ioContext = ioCxt;
+ }
+
private CloseableIterator<T> nextTask() {
FileScanTask task = tasks.next();
CloseableIterator<T> closeableIterator = open(task,
expectedSchema).iterator();
@@ -90,7 +103,7 @@ private CloseableIterator<T> nextTask() {
return closeableIterator;
}
return new IcebergAcidUtil.VirtualColumnAwareIterator<>(closeableIterator,
- expectedSchema.columns(), conf, task);
+ expectedSchema.columns(), task, () -> ioContext);
}
@Override
diff --git
a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergCRUD.java
b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergCRUD.java
index 87434a85201..e3236fe2c3e 100644
---
a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergCRUD.java
+++
b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergCRUD.java
@@ -81,8 +81,7 @@ private void createCopyOnWriteTable(TableIdentifier
identifier, String modePrope
@Test
public void testReadAndWriteFormatV2UnpartitionedWithEqDelete() throws
IOException {
- Assume.assumeTrue("Reading V2 tables with eq delete files are only
supported currently in " +
- "non-vectorized mode", !isVectorized && formatVersion == 2);
+ Assume.assumeTrue(formatVersion == 2);
Table tbl = testTables.createTable(shell, "customers",
HiveIcebergStorageHandlerTestUtils.CUSTOMER_SCHEMA,
PartitionSpec.unpartitioned(), fileFormat,
HiveIcebergStorageHandlerTestUtils.CUSTOMER_RECORDS, formatVersion);
@@ -104,8 +103,7 @@ public void
testReadAndWriteFormatV2UnpartitionedWithEqDelete() throws IOExcepti
@Test
public void
testReadAndWriteFormatV2Partitioned_EqDelete_AllColumnsSupplied() throws
IOException {
- Assume.assumeTrue("Reading V2 tables with eq delete files are only
supported currently in " +
- "non-vectorized mode", !isVectorized && formatVersion == 2);
+ Assume.assumeTrue(formatVersion == 2);
PartitionSpec spec =
PartitionSpec.builderFor(HiveIcebergStorageHandlerTestUtils.CUSTOMER_SCHEMA)
.identity("customer_id").build();
@@ -131,8 +129,7 @@ public void
testReadAndWriteFormatV2Partitioned_EqDelete_AllColumnsSupplied() th
@Test
public void
testReadAndWriteFormatV2Partitioned_EqDelete_OnlyEqColumnsSupplied() throws
IOException {
- Assume.assumeTrue("Reading V2 tables with eq delete files are only
supported currently in " +
- "non-vectorized mode", !isVectorized && formatVersion == 2);
+ Assume.assumeTrue(formatVersion == 2);
PartitionSpec spec =
PartitionSpec.builderFor(HiveIcebergStorageHandlerTestUtils.CUSTOMER_SCHEMA)
.identity("customer_id").build();
diff --git
a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergTimeTravel.java
b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergTimeTravel.java
index 6552b870560..38040c75147 100644
---
a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergTimeTravel.java
+++
b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergTimeTravel.java
@@ -22,11 +22,17 @@
import java.io.IOException;
import java.util.Collection;
import java.util.List;
+import org.apache.iceberg.DeleteFile;
import org.apache.iceberg.FileFormat;
import org.apache.iceberg.HistoryEntry;
+import org.apache.iceberg.PartitionSpec;
import org.apache.iceberg.Table;
+import org.apache.iceberg.data.Record;
+import org.apache.iceberg.mr.TestHelper;
import org.apache.iceberg.mr.hive.test.TestTables.TestTableType;
import
org.apache.iceberg.mr.hive.test.utils.HiveIcebergStorageHandlerTestUtils;
+import org.apache.iceberg.mr.hive.test.utils.HiveIcebergTestUtils;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
import org.apache.iceberg.types.Types;
import org.junit.Assert;
import org.junit.Test;
@@ -219,4 +225,27 @@ public void testAsOfWithJoins() throws IOException,
InterruptedException {
Assert.assertEquals(8, rows.size());
}
+
+ @Test
+ public void testSelectBranchTagAndAsOfWithEqualityDeletes() throws
IOException {
+ Table table = testTables.createTable(shell, "customers",
HiveIcebergStorageHandlerTestUtils.CUSTOMER_SCHEMA,
+ PartitionSpec.unpartitioned(), fileFormat,
HiveIcebergStorageHandlerTestUtils.CUSTOMER_RECORDS,
+ formatVersion);
+ List<Record> toDelete = TestHelper.RecordsBuilder
+
.newInstance(HiveIcebergStorageHandlerTestUtils.CUSTOMER_SCHEMA).add(1L, "Bob",
null).build();
+ DeleteFile deleteFile =
HiveIcebergTestUtils.createEqualityDeleteFile(table, "dummyPath",
+ ImmutableList.of("customer_id", "first_name"), fileFormat, toDelete);
+ table.newRowDelta().addDeletes(deleteFile).commit();
+ long snapshotId = table.currentSnapshot().snapshotId();
+ table.newRowDelta().removeDeletes(deleteFile).commit();
+ shell.executeStatement("ALTER TABLE customers CREATE BRANCH eq_branch FOR
SYSTEM_VERSION AS OF " + snapshotId);
+ shell.executeStatement("ALTER TABLE customers CREATE TAG eq_tag FOR
SYSTEM_VERSION AS OF " + snapshotId);
+
+ Assert.assertEquals(2, shell.executeStatement("SELECT * FROM
default.customers.branch_eq_branch").size());
+ Assert.assertEquals(2, shell.executeStatement("SELECT * FROM
default.customers.tag_eq_tag").size());
+ Assert.assertEquals(2,
+ shell.executeStatement("SELECT * FROM customers FOR SYSTEM_VERSION AS
OF " + snapshotId).size());
+ Assert.assertEquals(2, shell.executeStatement(
+ "SELECT * FROM customers FOR SYSTEM_TIME AS OF '" +
timestampAfterSnapshot(table, 1) + "'").size());
+ }
}
diff --git
a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_row_mode_virtual_columns.q
b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_row_mode_virtual_columns.q
new file mode 100644
index 00000000000..86dcf11710c
--- /dev/null
+++
b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_row_mode_virtual_columns.q
@@ -0,0 +1,20 @@
+-- SORT_QUERY_RESULTS
+-- row mode hands the Iceberg virtual columns to the operators through the
IOContext
+set hive.vectorized.execution.enabled=false;
+
+create table ice_vc (id int, p string) partitioned by spec (p) stored by
iceberg stored as parquet
+ tblproperties ('format-version'='3');
+
+insert into ice_vc values (1, 'a'), (2, 'a'), (3, 'b');
+insert into ice_vc values (4, 'b');
+
+select id, p, ROW__POSITION, PARTITION__SPEC__ID, PARTITION__NAME, FILE__PATH
is not null,
+ ROW__LINEAGE__ID, LAST__UPDATED__SEQUENCE__NUMBER
+from ice_vc order by id;
+
+-- a fetch task reads them through its own wrapper of the reader
+set hive.fetch.task.conversion=more;
+explain select id, ROW__POSITION, PARTITION__NAME, ROW__LINEAGE__ID from
ice_vc;
+select id, ROW__POSITION, PARTITION__NAME, ROW__LINEAGE__ID from ice_vc;
+
+drop table ice_vc;
diff --git a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_stats.q
b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_stats.q
index 6fc965e1745..e510157cef9 100644
--- a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_stats.q
+++ b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_stats.q
@@ -49,6 +49,34 @@ select count(*) from ice03;
insert overwrite table ice03 select * from ice03;
explain select count(*) from ice03;
+-- under the metastore source a plain ANALYZE counts the rows; the commit's
row count includes the deleted rows
+set hive.iceberg.stats.source=metastore;
+
+create external table ice07 (id int, key int) Stored by Iceberg stored as ORC
+ TBLPROPERTIES('format-version'='2');
+
+insert into ice07 values (1,1),(2,1),(3,1),(4,1),(5,1);
+delete from ice07 where id in (2,4);
+analyze table ice07 compute statistics;
+show tblproperties ice07('COLUMN_STATS_ACCURATE');
+show tblproperties ice07('numRows');
+select count(*) from ice07;
+
+-- with iceberg.hive.keep.stats the delete's commit keeps that row count
marked accurate; ANALYZE replaces it
+set iceberg.hive.keep.stats=true;
+
+create external table ice08 (id int, key int) Stored by Iceberg stored as ORC
+ TBLPROPERTIES('format-version'='2');
+
+insert into ice08 values (1,1),(2,1),(3,1),(4,1),(5,1);
+delete from ice08 where id in (2,4);
+analyze table ice08 compute statistics;
+show tblproperties ice08('COLUMN_STATS_ACCURATE');
+show tblproperties ice08('numRows');
+select count(*) from ice08;
+
drop table ice01;
drop table ice02;
drop table ice03;
+drop table ice07;
+drop table ice08;
diff --git
a/iceberg/iceberg-handler/src/test/queries/positive/vectorized_iceberg_read_parquet.q
b/iceberg/iceberg-handler/src/test/queries/positive/vectorized_iceberg_read_parquet.q
index 38ef8305a80..ce1d46c07f2 100644
---
a/iceberg/iceberg-handler/src/test/queries/positive/vectorized_iceberg_read_parquet.q
+++
b/iceberg/iceberg-handler/src/test/queries/positive/vectorized_iceberg_read_parquet.q
@@ -4,6 +4,8 @@ drop table if exists tbl_ice_parquet;
create external table tbl_ice_parquet(a int, b string) stored by iceberg
stored as parquet
TBLPROPERTIES ("format-version"='1');
insert into table tbl_ice_parquet values (1, 'one'), (2, 'two'), (3, 'three'),
(4, 'four'), (5, 'five'), (111, 'one'), (22, 'two'), (11, 'one'), (44444,
'four'), (44, 'four');
+-- the storage handler provides the basic statistics, so the scan gathers none
and runs vectorized
+explain vectorization only analyze table tbl_ice_parquet compute statistics
for columns;
analyze table tbl_ice_parquet compute statistics for columns;
explain select b, max(a) from tbl_ice_parquet group by b;
diff --git
a/iceberg/iceberg-handler/src/test/results/positive/iceberg_stats.q.out
b/iceberg/iceberg-handler/src/test/results/positive/iceberg_stats.q.out
index 4e5b7094501..cc88f9b788a 100644
--- a/iceberg/iceberg-handler/src/test/results/positive/iceberg_stats.q.out
+++ b/iceberg/iceberg-handler/src/test/results/positive/iceberg_stats.q.out
@@ -341,6 +341,112 @@ STAGE PLANS:
Processor Tree:
ListSink
+PREHOOK: query: create external table ice07 (id int, key int) Stored by
Iceberg stored as ORC
+ TBLPROPERTIES('format-version'='2')
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@ice07
+POSTHOOK: query: create external table ice07 (id int, key int) Stored by
Iceberg stored as ORC
+ TBLPROPERTIES('format-version'='2')
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@ice07
+PREHOOK: query: insert into ice07 values (1,1),(2,1),(3,1),(4,1),(5,1)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@ice07
+POSTHOOK: query: insert into ice07 values (1,1),(2,1),(3,1),(4,1),(5,1)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@ice07
+PREHOOK: query: delete from ice07 where id in (2,4)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@ice07
+PREHOOK: Output: default@ice07
+POSTHOOK: query: delete from ice07 where id in (2,4)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@ice07
+POSTHOOK: Output: default@ice07
+PREHOOK: query: analyze table ice07 compute statistics
+PREHOOK: type: QUERY
+PREHOOK: Input: default@ice07
+PREHOOK: Output: default@ice07
+POSTHOOK: query: analyze table ice07 compute statistics
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@ice07
+POSTHOOK: Output: default@ice07
+PREHOOK: query: show tblproperties ice07('COLUMN_STATS_ACCURATE')
+PREHOOK: type: SHOW_TBLPROPERTIES
+POSTHOOK: query: show tblproperties ice07('COLUMN_STATS_ACCURATE')
+POSTHOOK: type: SHOW_TBLPROPERTIES
+COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+PREHOOK: query: show tblproperties ice07('numRows')
+PREHOOK: type: SHOW_TBLPROPERTIES
+POSTHOOK: query: show tblproperties ice07('numRows')
+POSTHOOK: type: SHOW_TBLPROPERTIES
+numRows 3
+PREHOOK: query: select count(*) from ice07
+PREHOOK: type: QUERY
+PREHOOK: Input: default@ice07
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: select count(*) from ice07
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@ice07
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+3
+PREHOOK: query: create external table ice08 (id int, key int) Stored by
Iceberg stored as ORC
+ TBLPROPERTIES('format-version'='2')
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@ice08
+POSTHOOK: query: create external table ice08 (id int, key int) Stored by
Iceberg stored as ORC
+ TBLPROPERTIES('format-version'='2')
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@ice08
+PREHOOK: query: insert into ice08 values (1,1),(2,1),(3,1),(4,1),(5,1)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@ice08
+POSTHOOK: query: insert into ice08 values (1,1),(2,1),(3,1),(4,1),(5,1)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@ice08
+PREHOOK: query: delete from ice08 where id in (2,4)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@ice08
+PREHOOK: Output: default@ice08
+POSTHOOK: query: delete from ice08 where id in (2,4)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@ice08
+POSTHOOK: Output: default@ice08
+PREHOOK: query: analyze table ice08 compute statistics
+PREHOOK: type: QUERY
+PREHOOK: Input: default@ice08
+PREHOOK: Output: default@ice08
+POSTHOOK: query: analyze table ice08 compute statistics
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@ice08
+POSTHOOK: Output: default@ice08
+PREHOOK: query: show tblproperties ice08('COLUMN_STATS_ACCURATE')
+PREHOOK: type: SHOW_TBLPROPERTIES
+POSTHOOK: query: show tblproperties ice08('COLUMN_STATS_ACCURATE')
+POSTHOOK: type: SHOW_TBLPROPERTIES
+COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+PREHOOK: query: show tblproperties ice08('numRows')
+PREHOOK: type: SHOW_TBLPROPERTIES
+POSTHOOK: query: show tblproperties ice08('numRows')
+POSTHOOK: type: SHOW_TBLPROPERTIES
+numRows 3
+PREHOOK: query: select count(*) from ice08
+PREHOOK: type: QUERY
+PREHOOK: Input: default@ice08
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: select count(*) from ice08
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@ice08
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+3
PREHOOK: query: drop table ice01
PREHOOK: type: DROPTABLE
PREHOOK: Input: default@ice01
@@ -371,3 +477,23 @@ POSTHOOK: type: DROPTABLE
POSTHOOK: Input: default@ice03
POSTHOOK: Output: database:default
POSTHOOK: Output: default@ice03
+PREHOOK: query: drop table ice07
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@ice07
+PREHOOK: Output: database:default
+PREHOOK: Output: default@ice07
+POSTHOOK: query: drop table ice07
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@ice07
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@ice07
+PREHOOK: query: drop table ice08
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@ice08
+PREHOOK: Output: database:default
+PREHOOK: Output: default@ice08
+POSTHOOK: query: drop table ice08
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@ice08
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@ice08
diff --git
a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_row_mode_virtual_columns.q.out
b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_row_mode_virtual_columns.q.out
new file mode 100644
index 00000000000..4ad0ee0d969
--- /dev/null
+++
b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_row_mode_virtual_columns.q.out
@@ -0,0 +1,87 @@
+PREHOOK: query: create table ice_vc (id int, p string) partitioned by spec (p)
stored by iceberg stored as parquet
+ tblproperties ('format-version'='3')
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@ice_vc
+POSTHOOK: query: create table ice_vc (id int, p string) partitioned by spec
(p) stored by iceberg stored as parquet
+ tblproperties ('format-version'='3')
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@ice_vc
+PREHOOK: query: insert into ice_vc values (1, 'a'), (2, 'a'), (3, 'b')
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@ice_vc
+POSTHOOK: query: insert into ice_vc values (1, 'a'), (2, 'a'), (3, 'b')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@ice_vc
+PREHOOK: query: insert into ice_vc values (4, 'b')
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@ice_vc
+POSTHOOK: query: insert into ice_vc values (4, 'b')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@ice_vc
+PREHOOK: query: select id, p, ROW__POSITION, PARTITION__SPEC__ID,
PARTITION__NAME, FILE__PATH is not null,
+ ROW__LINEAGE__ID, LAST__UPDATED__SEQUENCE__NUMBER
+from ice_vc order by id
+PREHOOK: type: QUERY
+PREHOOK: Input: default@ice_vc
+#### A masked pattern was here ####
+POSTHOOK: query: select id, p, ROW__POSITION, PARTITION__SPEC__ID,
PARTITION__NAME, FILE__PATH is not null,
+ ROW__LINEAGE__ID, LAST__UPDATED__SEQUENCE__NUMBER
+from ice_vc order by id
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@ice_vc
+#### A masked pattern was here ####
+1 a 0 0 p=a true 0 1
+2 a 1 0 p=a true 1 1
+3 b 0 0 p=b true 2 1
+4 b 0 0 p=b true 3 2
+PREHOOK: query: explain select id, ROW__POSITION, PARTITION__NAME,
ROW__LINEAGE__ID from ice_vc
+PREHOOK: type: QUERY
+PREHOOK: Input: default@ice_vc
+#### A masked pattern was here ####
+POSTHOOK: query: explain select id, ROW__POSITION, PARTITION__NAME,
ROW__LINEAGE__ID from ice_vc
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@ice_vc
+#### A masked pattern was here ####
+STAGE DEPENDENCIES:
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ TableScan
+ alias: ice_vc
+ Select Operator
+ expressions: id (type: int), ROW__POSITION (type: bigint),
PARTITION__NAME (type: string), ROW__LINEAGE__ID (type: bigint)
+ outputColumnNames: _col0, _col1, _col2, _col3
+ ListSink
+
+PREHOOK: query: select id, ROW__POSITION, PARTITION__NAME, ROW__LINEAGE__ID
from ice_vc
+PREHOOK: type: QUERY
+PREHOOK: Input: default@ice_vc
+#### A masked pattern was here ####
+POSTHOOK: query: select id, ROW__POSITION, PARTITION__NAME, ROW__LINEAGE__ID
from ice_vc
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@ice_vc
+#### A masked pattern was here ####
+1 0 p=a 0
+2 1 p=a 1
+3 0 p=b 2
+4 0 p=b 3
+PREHOOK: query: drop table ice_vc
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@ice_vc
+PREHOOK: Output: database:default
+PREHOOK: Output: default@ice_vc
+POSTHOOK: query: drop table ice_vc
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@ice_vc
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@ice_vc
diff --git
a/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_parquet.q.out
b/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_parquet.q.out
index 7e9983b34fc..d92a34783d9 100644
---
a/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_parquet.q.out
+++
b/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_parquet.q.out
@@ -22,6 +22,53 @@ POSTHOOK: query: insert into table tbl_ice_parquet values
(1, 'one'), (2, 'two')
POSTHOOK: type: QUERY
POSTHOOK: Input: _dummy_database@_dummy_table
POSTHOOK: Output: default@tbl_ice_parquet
+PREHOOK: query: explain vectorization only analyze table tbl_ice_parquet
compute statistics for columns
+PREHOOK: type: ANALYZE_TABLE
+PREHOOK: Input: default@tbl_ice_parquet
+PREHOOK: Output: default@tbl_ice_parquet
+#### A masked pattern was here ####
+POSTHOOK: query: explain vectorization only analyze table tbl_ice_parquet
compute statistics for columns
+POSTHOOK: type: ANALYZE_TABLE
+POSTHOOK: Input: default@tbl_ice_parquet
+POSTHOOK: Output: default@tbl_ice_parquet
+#### A masked pattern was here ####
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
+STAGE DEPENDENCIES:
+ Stage-0 is a root stage
+ Stage-2 depends on stages: Stage-0
+
+STAGE PLANS:
+ Stage: Stage-0
+ Tez
+ Edges:
+ Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE)
+ Vertices:
+ Map 1
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs (cache only)
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet:
hive.vectorized.use.vectorized.input.format IS true
+ inputFormatFeatureSupport: [DECIMAL_64]
+ featureSupportInUse: [DECIMAL_64]
+ inputFileFormats:
org.apache.iceberg.mr.hive.HiveIcebergInputFormat
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ Reducer 2
+ Execution mode: vectorized, llap
+ Reduce Vectorization:
+ enabled: true
+ enableConditionsMet: hive.vectorized.execution.reduce.enabled
IS true, hive.execution.engine tez IN [tez] IS true
+ allNative: false
+ usesVectorUDFAdaptor: true
+ vectorized: true
+
+ Stage: Stage-2
+
PREHOOK: query: analyze table tbl_ice_parquet compute statistics for columns
PREHOOK: type: ANALYZE_TABLE
PREHOOK: Input: default@tbl_ice_parquet
diff --git
a/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_parquet.q.out
b/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_parquet.q.out
index 4b4d114a151..ca9f92e58a0 100644
---
a/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_parquet.q.out
+++
b/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_parquet.q.out
@@ -22,6 +22,52 @@ POSTHOOK: query: insert into table tbl_ice_parquet values
(1, 'one'), (2, 'two')
POSTHOOK: type: QUERY
POSTHOOK: Input: _dummy_database@_dummy_table
POSTHOOK: Output: default@tbl_ice_parquet
+PREHOOK: query: explain vectorization only analyze table tbl_ice_parquet
compute statistics for columns
+PREHOOK: type: ANALYZE_TABLE
+PREHOOK: Input: default@tbl_ice_parquet
+PREHOOK: Output: default@tbl_ice_parquet
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: explain vectorization only analyze table tbl_ice_parquet
compute statistics for columns
+POSTHOOK: type: ANALYZE_TABLE
+POSTHOOK: Input: default@tbl_ice_parquet
+POSTHOOK: Output: default@tbl_ice_parquet
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
+STAGE DEPENDENCIES:
+ Stage-0 is a root stage
+ Stage-2 depends on stages: Stage-0
+
+STAGE PLANS:
+ Stage: Stage-0
+ Tez
+ Edges:
+ Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE)
+ Vertices:
+ Map 1
+ Execution mode: vectorized
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet:
hive.vectorized.use.vectorized.input.format IS true
+ inputFormatFeatureSupport: [DECIMAL_64]
+ featureSupportInUse: [DECIMAL_64]
+ inputFileFormats:
org.apache.iceberg.mr.hive.HiveIcebergInputFormat
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ Reducer 2
+ Execution mode: vectorized
+ Reduce Vectorization:
+ enabled: true
+ enableConditionsMet: hive.vectorized.execution.reduce.enabled
IS true, hive.execution.engine tez IN [tez] IS true
+ allNative: false
+ usesVectorUDFAdaptor: true
+ vectorized: true
+
+ Stage: Stage-2
+
PREHOOK: query: analyze table tbl_ice_parquet compute statistics for columns
PREHOOK: type: ANALYZE_TABLE
PREHOOK: Input: default@tbl_ice_parquet
diff --git a/itests/src/test/resources/testconfiguration.properties
b/itests/src/test/resources/testconfiguration.properties
index 575d027fc02..a0f1e4b4539 100644
--- a/itests/src/test/resources/testconfiguration.properties
+++ b/itests/src/test/resources/testconfiguration.properties
@@ -419,6 +419,7 @@ iceberg.llap.query.files=\
iceberg_create_locally_zordered_table.q,\
iceberg_merge_delete_files.q,\
iceberg_merge_files.q,\
+ iceberg_row_mode_virtual_columns.q,\
llap_iceberg_read_orc.q,\
llap_iceberg_read_parquet.q,\
puffin_col_stats_with_time_travel.q,\
@@ -478,6 +479,7 @@ iceberg.llap.only.query.files=\
iceberg_create_locally_zordered_table.q,\
iceberg_merge_delete_files.q,\
iceberg_merge_files.q,\
+ iceberg_row_mode_virtual_columns.q,\
llap_iceberg_read_orc.q,\
llap_iceberg_read_parquet.q,\
puffin_col_stats_with_time_travel.q
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
index ea7603d39c7..21daa8a2554 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
@@ -262,6 +262,8 @@ public final class Utilities {
public static final String ENSURE_OPERATORS_EXECUTED =
"ENSURE_OPERATORS_EXECUTED";
public static final String CREATE_TIME = "create_time";
public static final String SNAPSHOT_REF = "snapshot_ref";
+ public static final String AS_OF_VERSION = "as_of_version";
+ public static final String AS_OF_TIMESTAMP = "as_of_timestamp";
@Deprecated
protected static final String DEPRECATED_MAPRED_DFSCLIENT_PARALLELISM_MAX =
"mapred.dfsclient.parallelism.max";
@@ -771,6 +773,12 @@ public static TableDesc getTableDesc(Table tbl) {
if (tbl.getSnapshotRef() != null) {
props.put(SNAPSHOT_REF, tbl.getSnapshotRef());
}
+ if (tbl.getAsOfVersion() != null) {
+ props.put(AS_OF_VERSION, tbl.getAsOfVersion());
+ }
+ if (tbl.getAsOfTimestamp() != null) {
+ props.put(AS_OF_TIMESTAMP, tbl.getAsOfTimestamp());
+ }
return new TableDesc(tbl.getInputFormatClass(),
tbl.getOutputFormatClass(), props, tbl.getCatName());
}
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/io/HiveContextAwareRecordReader.java
b/ql/src/java/org/apache/hadoop/hive/ql/io/HiveContextAwareRecordReader.java
index f29473af340..6f941066e94 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/HiveContextAwareRecordReader.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/HiveContextAwareRecordReader.java
@@ -134,8 +134,6 @@ else if(recordReader instanceof
AcidInputFormat.AcidRecordReader) {
ioCxtRef.setDeletedRecord(recordIdentifier != null &&
recordIdentifier.isDeleteEvent());
} else {
ioCxtRef.parseRecordIdentifier(jobConf);
- ioCxtRef.parsePositionDeleteInfo(jobConf);
- ioCxtRef.parseRowLineageInfo(jobConf);
}
}
return retVal;
@@ -182,6 +180,9 @@ public IOContext getIOContext() {
private void initIOContext(long startPos, boolean isBlockPointer,
Path inputPath) {
ioCxtRef = this.getIOContext();
+ if (recordReader instanceof IOContextAwareRecordReader reader) {
+ reader.setIOContext(ioCxtRef);
+ }
ioCxtRef.setCurrentBlockStart(startPos);
ioCxtRef.setBlockPointer(isBlockPointer);
ioCxtRef.setInputPath(inputPath);
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/IOContext.java
b/ql/src/java/org/apache/hadoop/hive/ql/io/IOContext.java
index 0dd8b3afb29..15947e6e34e 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/IOContext.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/IOContext.java
@@ -21,7 +21,6 @@
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.mapred.JobConf;
/**
* IOContext basically contains the position information of the current
@@ -194,8 +193,8 @@ public void parseRecordIdentifier(Configuration
configuration) {
}
}
- public void parsePositionDeleteInfo(Configuration configuration) {
- this.pdi = PositionDeleteInfo.parseFromConf(configuration);
+ public void setPositionDeleteInfo(PositionDeleteInfo pdi) {
+ this.pdi = pdi;
}
public PositionDeleteInfo getPositionDeleteInfo() {
@@ -210,8 +209,8 @@ public String getPartitionName() {
return partitionName;
}
- public void parseRowLineageInfo(JobConf cconfiguration) {
- this.rowLineageInfo = RowLineageInfo.parseFromConf(cconfiguration);
+ public void setRowLineageInfo(RowLineageInfo rowLineageInfo) {
+ this.rowLineageInfo = rowLineageInfo;
}
public RowLineageInfo getRowLineageInfo() {
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/io/IOContextAwareRecordReader.java
b/ql/src/java/org/apache/hadoop/hive/ql/io/IOContextAwareRecordReader.java
new file mode 100644
index 00000000000..6a064b2cf91
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/IOContextAwareRecordReader.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hadoop.hive.ql.io;
+
+/**
+ * A record reader that writes the virtual columns of the rows it returns into
an IOContext.
+ */
+public interface IOContextAwareRecordReader {
+
+ /**
+ * Sets the IOContext the virtual columns of each row this reader returns
are written into. The
+ * HiveContextAwareRecordReader wrapping this reader, directly or through
another reader, calls it when it binds
+ * its IOContext, before the first row.
+ */
+ void setIOContext(IOContext ioContext);
+}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/PositionDeleteInfo.java
b/ql/src/java/org/apache/hadoop/hive/ql/io/PositionDeleteInfo.java
index d3485a49f71..d069c405653 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/PositionDeleteInfo.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/PositionDeleteInfo.java
@@ -19,30 +19,8 @@
package org.apache.hadoop.hive.ql.io;
-import org.apache.hadoop.conf.Configuration;
-
public class PositionDeleteInfo {
- private static final String CONF_KEY_SPEC_ID =
"hive.io.context.position.delete.spec.id";
- private static final String CONF_KEY_PART_HASH =
"hive.io.context.position.delete.partition.hash";
- private static final String CONF_KEY_FILE_PATH =
"hive.io.context.position.delete.file.path";
- private static final String CONF_KEY_ROW_POSITION =
"hive.io.context.position.delete.row.position";
-
- public static PositionDeleteInfo parseFromConf(Configuration conf) {
- int specId = conf.getInt(CONF_KEY_SPEC_ID, -1);
- long partHash = conf.getLong(CONF_KEY_PART_HASH, -1);
- String filePath = conf.get(CONF_KEY_FILE_PATH);
- long rowPos = conf.getLong(CONF_KEY_ROW_POSITION, -1);
- return new PositionDeleteInfo(specId, partHash, filePath, rowPos);
- }
-
- public static void setIntoConf(Configuration conf, int specId, long
partHash, String filePath, long filePos) {
- conf.setInt(CONF_KEY_SPEC_ID, specId);
- conf.setLong(CONF_KEY_PART_HASH, partHash);
- conf.set(CONF_KEY_FILE_PATH, filePath);
- conf.setLong(CONF_KEY_ROW_POSITION, filePos);
- }
-
private final int specId;
private final long partitionHash;
private final String filePath;
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/RowLineageInfo.java
b/ql/src/java/org/apache/hadoop/hive/ql/io/RowLineageInfo.java
index 6433d0f36e9..5bc03cfdc91 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/RowLineageInfo.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/RowLineageInfo.java
@@ -19,17 +19,11 @@
package org.apache.hadoop.hive.ql.io;
-import org.apache.hadoop.conf.Configuration;
-
public class RowLineageInfo {
private final Long baseRowId;
private final Long lastUpdatedSequenceNumber;
- private static final String CONF_KEY_ROW_ID =
"hive.io.context.row.lineage.row.id";
- private static final String CONF_KEY_LAST_UPDATED_SEQUENCE_NUMBER =
- "hive.io.context.row.lineage.last.updated.sequence.number";
-
public RowLineageInfo(Long baseRowId, Long lastUpdatedSequenceNumber) {
this.baseRowId = baseRowId;
this.lastUpdatedSequenceNumber = lastUpdatedSequenceNumber;
@@ -42,23 +36,4 @@ public Long getBaseRowId() {
public Long getLastUpdatedSequenceNumber() {
return lastUpdatedSequenceNumber;
}
-
- public static RowLineageInfo parseFromConf(Configuration conf) {
- Long rowId = conf.get(CONF_KEY_ROW_ID) != null ?
Long.parseLong(conf.get(CONF_KEY_ROW_ID)) : null;
-
- Long lusn = conf.get(CONF_KEY_LAST_UPDATED_SEQUENCE_NUMBER) != null ?
- Long.parseLong(conf.get(CONF_KEY_LAST_UPDATED_SEQUENCE_NUMBER)) :
- null;
-
- return new RowLineageInfo(rowId, lusn);
- }
-
- public static void setRowLineageInfoIntoConf(Long rowId, Long lusn,
Configuration conf) {
- if (rowId != null) {
- conf.setLong(CONF_KEY_ROW_ID, rowId);
- }
- if (lusn != null) {
- conf.setLong(CONF_KEY_LAST_UPDATED_SEQUENCE_NUMBER, lusn);
- }
- }
}
\ No newline at end of file
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveStorageHandler.java
b/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveStorageHandler.java
index 472a884293c..20f62c015f2 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveStorageHandler.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveStorageHandler.java
@@ -269,13 +269,24 @@ default Map<String, String>
computeBasicStatistics(org.apache.hadoop.hive.ql.met
}
/**
- * Check if the storage handler can provide basic statistics.
+ * Check if the storage handler can provide basic statistics, whether it
keeps them itself (see
+ * {@link
#canProvideBasicStatistics(org.apache.hadoop.hive.ql.metadata.Table)}) or reads
them from the metastore.
* @return true if the storage handler can supply the basic statistics
*/
default boolean canProvideBasicStatistics() {
return false;
}
-
+
+ /**
+ * Check if the storage handler supplies the table's basic statistics
itself, rather than from the metastore
+ * parameters an ANALYZE scan writes.
+ * @param table table object
+ * @return true if the storage handler supplies the table's basic statistics
+ */
+ default boolean
canProvideBasicStatistics(org.apache.hadoop.hive.ql.metadata.Table table) {
+ return canProvideBasicStatistics();
+ }
+
/**
* Check if the storage handler can provide partition statistics.
* @return true if the storage handler can supply the partition statistics
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
index cadb097c4d4..bb531aff14d 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
@@ -266,6 +266,7 @@
import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hadoop.hive.ql.session.SessionState.ResourceType;
import org.apache.hadoop.hive.ql.session.SessionStateUtil;
+import org.apache.hadoop.hive.ql.stats.BasicStatsNoJobTask;
import org.apache.hadoop.hive.ql.stats.StatsUtils;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator.Mode;
@@ -12347,34 +12348,11 @@ private void setupStats(TableScanDesc tsDesc,
QBParseInfo qbp, Table tab, String
return;
}
- if (HiveConf.getVar(conf,
HIVE_STATS_DBCLASS).equalsIgnoreCase(StatDB.fs.name())) {
- String statsTmpLoc =
ctx.getTempDirForInterimJobPath(tab.getPath()).toString();
- LOG.debug("Set stats collection dir : " + statsTmpLoc);
- tsDesc.setTmpStatsDir(statsTmpLoc);
- }
- tsDesc.setGatherStats(true);
-
tsDesc.setStatsReliable(conf.getBoolVar(HiveConf.ConfVars.HIVE_STATS_RELIABLE));
-
- // append additional virtual columns for storing statistics
- Iterator<VirtualColumn> vcs =
VirtualColumn.getStatsRegistry(conf).iterator();
- List<VirtualColumn> vcList = new ArrayList<VirtualColumn>();
- while (vcs.hasNext()) {
- VirtualColumn vc = vcs.next();
- rwsch.put(alias, vc.getName(), new ColumnInfo(vc.getName(),
- vc.getTypeInfo(), alias, true, vc.getIsHidden()));
- vcList.add(vc);
+ // basic statistics the storage handler keeps make scan-side gathering
redundant, and it keeps the scan from
+ // vectorizing
+ if (!BasicStatsNoJobTask.storageHandlerOwnsBasicStats(tab)) {
+ setupStatsGathering(tsDesc, tab, alias, rwsch);
}
- tsDesc.addVirtualCols(vcList);
-
- String tblName = tab.getTableName();
- // Theoretically the key prefix could be any unique string shared
- // between TableScanOperator (when publishing) and StatsTask (when
aggregating).
- // Here we use
- // db_name.table_name + partitionSec
- // as the prefix for easy of read during explain and debugging.
- // Currently, partition spec can only be static partition.
- String k = FileUtils.escapePathName(tblName).toLowerCase() +
Path.SEPARATOR;
-
tsDesc.setStatsAggPrefix(FileUtils.escapePathName(tab.getDbName()).toLowerCase()
+ "." + k);
// set up WriteEntity for replication and txn stats
WriteEntity we = new WriteEntity(tab, WriteEntity.WriteType.DDL_SHARED);
@@ -12418,6 +12396,37 @@ private void setupStats(TableScanDesc tsDesc,
QBParseInfo qbp, Table tab, String
}
}
+ private void setupStatsGathering(TableScanDesc tsDesc, Table tab, String
alias, RowResolver rwsch) {
+ if (HiveConf.getVar(conf,
HIVE_STATS_DBCLASS).equalsIgnoreCase(StatDB.fs.name())) {
+ String statsTmpLoc =
ctx.getTempDirForInterimJobPath(tab.getPath()).toString();
+ LOG.debug("Set stats collection dir : " + statsTmpLoc);
+ tsDesc.setTmpStatsDir(statsTmpLoc);
+ }
+ tsDesc.setGatherStats(true);
+
tsDesc.setStatsReliable(conf.getBoolVar(HiveConf.ConfVars.HIVE_STATS_RELIABLE));
+
+ // append additional virtual columns for storing statistics
+ Iterator<VirtualColumn> vcs =
VirtualColumn.getStatsRegistry(conf).iterator();
+ List<VirtualColumn> vcList = new ArrayList<>();
+ while (vcs.hasNext()) {
+ VirtualColumn vc = vcs.next();
+ rwsch.put(alias, vc.getName(), new ColumnInfo(vc.getName(),
+ vc.getTypeInfo(), alias, true, vc.getIsHidden()));
+ vcList.add(vc);
+ }
+ tsDesc.addVirtualCols(vcList);
+
+ String tblName = tab.getTableName();
+ // Theoretically the key prefix could be any unique string shared
+ // between TableScanOperator (when publishing) and StatsTask (when
aggregating).
+ // Here we use
+ // db_name.table_name + partitionSec
+ // as the prefix for easy of read during explain and debugging.
+ // Currently, partition spec can only be static partition.
+ String k = FileUtils.escapePathName(tblName).toLowerCase() +
Path.SEPARATOR;
+
tsDesc.setStatsAggPrefix(FileUtils.escapePathName(tab.getDbName()).toLowerCase()
+ "." + k);
+ }
+
private Operator genPlan(QB parent, QBExpr qbexpr) throws SemanticException {
if (qbexpr.getOpcode() == QBExpr.Opcode.EXCEPT || qbexpr.getOpcode() ==
QBExpr.Opcode.EXCEPTALL
|| qbexpr.getOpcode() == QBExpr.Opcode.INTERSECT || qbexpr.getOpcode()
== QBExpr.Opcode.INTERSECTALL) {
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java
b/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java
index 185e8117761..4910bc3fcdc 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java
@@ -480,12 +480,12 @@ private Task<?> genTableStats(ParseContext parseContext,
TableScanOperator table
TableSpec tableSpec = new TableSpec(table, partitions);
tableScan.getConf().getTableMetadata().setTableSpec(tableSpec);
- if (BasicStatsNoJobTask.canUseFooterScan(table, inputFormat)) {
- // For ORC, there is no Tez Job for table stats.
+ if (BasicStatsNoJobTask.canUseBasicStats(table, inputFormat)) {
+ // basic stats come from the file footers or the storage handler, not
the scan
StatsWork columnStatsWork = new StatsWork(table, parseContext.getConf());
columnStatsWork.setFooterScan();
// If partition is specified, get pruned partition list
- if (partitions.size() > 0) {
+ if (!partitions.isEmpty()) {
columnStatsWork.addInputPartitions(parseContext.getPrunedPartitions(tableScan).getPartitions());
}
return TaskFactory.get(columnStatsWork);
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/stats/BasicStatsNoJobTask.java
b/ql/src/java/org/apache/hadoop/hive/ql/stats/BasicStatsNoJobTask.java
index c39ef11218c..5fa35ac7a7c 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/stats/BasicStatsNoJobTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/stats/BasicStatsNoJobTask.java
@@ -102,7 +102,7 @@ public BasicStatsNoJobTask(HiveConf conf,
BasicStatsNoJobWork work) {
public static boolean canUseBasicStats(
Table table, Class<? extends InputFormat> inputFormat) {
- return canUseFooterScan(table, inputFormat) ||
useBasicStatsFromStorageHandler(table);
+ return canUseFooterScan(table, inputFormat) ||
storageHandlerOwnsBasicStats(table);
}
public static boolean canUseFooterScan(Table table, Class<? extends
InputFormat> inputFormat) {
@@ -110,8 +110,8 @@ public static boolean canUseFooterScan(Table table, Class<?
extends InputFormat>
|| MapredParquetInputFormat.class.isAssignableFrom(inputFormat);
}
- private static boolean useBasicStatsFromStorageHandler(Table table) {
- return table.isNonNative() &&
table.getStorageHandler().canProvideBasicStatistics();
+ public static boolean storageHandlerOwnsBasicStats(Table table) {
+ return table.isNonNative() &&
table.getStorageHandler().canProvideBasicStatistics(table);
}
@@ -127,7 +127,7 @@ public int process(Hive db, Table tbl) throws Exception {
ExecutorService threadPool = StatsTask.newThreadPool(conf);
- return aggregateStats(threadPool, db);
+ return aggregateStats(threadPool, db, tbl);
}
public StageType getType() {
@@ -175,12 +175,6 @@ public HiveStorageHandlerStatCollector(Partish partish) {
public void run() {
try {
Table table = partish.getTable();
- if (partish.getPartition() != null) {
- // the storage handler maintains basic statistics for the table as a
whole, so no
- // statement produces a partition partish for it here
- throw new IllegalStateException("Partition-scoped statistics
collection is not supported for "
- + table.getFullyQualifiedName());
- }
Map<String, String> parameters = table.getParameters();
// computes and publishes the statistics the storage handler maintains
(e.g. the Iceberg
// partition stats file) before reading them back: ANALYZE catches up
on writes that
@@ -345,9 +339,12 @@ public void run() {
}
private Collection<Partition> getPartitions(Table table) {
+ if (table.hasNonNativePartitionSupport()) {
+ return null;
+ }
Collection<Partition> partitions = null;
if (work.getPartitions() == null || work.getPartitions().isEmpty()) {
- if (table.isPartitioned() && !table.hasNonNativePartitionSupport()) {
+ if (table.isPartitioned()) {
partitions = table.getTableSpec().partitions;
}
} else {
@@ -370,7 +367,7 @@ private List<Partish> getPartishes(Table table) {
}
- private int aggregateStats(ExecutorService threadPool, Hive db) {
+ private int aggregateStats(ExecutorService threadPool, Hive db, Table tbl) {
int ret = 0;
try {
JobConf jc = new JobConf(conf);
@@ -387,7 +384,7 @@ private int aggregateStats(ExecutorService threadPool, Hive
db) {
List<StatCollector> scs = new ArrayList();
for (Partish partish : partishes) {
- if (useBasicStatsFromStorageHandler(table)) {
+ if (storageHandlerOwnsBasicStats(table)) {
scs.add(new HiveStorageHandlerStatCollector(partish));
} else {
scs.add(new FooterStatCollector(jc, partish));
@@ -403,7 +400,19 @@ private int aggregateStats(ExecutorService threadPool,
Hive db) {
shutdownAndAwaitTermination(threadPool);
LOG.debug("Stats collection threadpool shutdown successful.");
- ret = updatePartitions(db, scs, table);
+ if (work.isStatsReliable()) {
+ for (StatCollector statsCollection : scs) {
+ if (!statsCollection.isValid()) {
+ LOG.debug("Stats requested to be reliable. Empty stats found: {}",
statsCollection.partish.getSimpleName());
+ return -1;
+ }
+ }
+ }
+ Table newTbl = updatePartitions(db, scs, table);
+ if (newTbl != null) {
+ // ColStatsProcessor writes the task's table object back: hand it the
parameters just written
+ tbl.setTTable(newTbl.getTTable());
+ }
} catch (Exception e) {
console.printError("Failed to collect footer statistics.", "Failed with
exception " + e.getMessage() + "\n" + StringUtils.stringifyException(e));
@@ -418,20 +427,16 @@ private int aggregateStats(ExecutorService threadPool,
Hive db) {
return ret;
}
- private int updatePartitions(Hive db, List<StatCollector> scs, Table table)
throws InvalidOperationException, HiveException {
+ /**
+ * @return the table whose statistics were written, or null
+ */
+ private Table updatePartitions(Hive db, List<StatCollector> scs, Table table)
+ throws InvalidOperationException, HiveException {
String tableFullName = table.getFullyQualifiedName();
if (scs.isEmpty()) {
- return 0;
- }
- if (work.isStatsReliable()) {
- for (StatCollector statsCollection : scs) {
- if (!statsCollection.isValid()) {
- LOG.debug("Stats requested to be reliable. Empty stats found: {}",
statsCollection.partish.getSimpleName());
- return -1;
- }
- }
+ return null;
}
List<StatCollector> validColectors = Lists.newArrayList();
for (StatCollector statsCollection : scs) {
@@ -456,6 +461,7 @@ private int updatePartitions(Hive db, List<StatCollector>
scs, Table table) thro
LOG.debug("Updating stats for: {}", tableFullName);
+ Table newTbl = null;
for (String partName : collectorsByTable.keySet()) {
ImmutableList<StatCollector> values = collectorsByTable.get(partName);
@@ -463,11 +469,12 @@ private int updatePartitions(Hive db, List<StatCollector>
scs, Table table) thro
throw new RuntimeException("very interesting");
}
- if (values.get(0).result instanceof Table) {
+ if (values.get(0).result instanceof Table result) {
// the metastore keeps one set of counts, and they describe the table,
not a branch
if (table.getSnapshotRef() == null) {
- db.alterTable(tableFullName, (Table) values.get(0).result,
environmentContext, true);
+ db.alterTable(tableFullName, result, environmentContext, true);
LOG.debug("Updated stats for {}.", tableFullName);
+ newTbl = result;
}
} else {
if (values.get(0).result instanceof Partition) {
@@ -480,7 +487,7 @@ private int updatePartitions(Hive db, List<StatCollector>
scs, Table table) thro
}
}
LOG.debug("Updated stats for: {}", tableFullName);
- return 0;
+ return newTbl;
}
private void shutdownAndAwaitTermination(ExecutorService threadPool) {
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/stats/BasicStatsTask.java
b/ql/src/java/org/apache/hadoop/hive/ql/stats/BasicStatsTask.java
index 9622d79ba40..ea30d259c7d 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/stats/BasicStatsTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/stats/BasicStatsTask.java
@@ -144,8 +144,8 @@ public BasicStatsProcessor(Partish partish, BasicStatsWork
work, boolean followe
if (table.isNonNative() &&
table.getStorageHandler().canProvideBasicStatistics()
&& partish.getPartition() == null) {
this.providedBasicStats =
table.getStorageHandler().computeBasicStatistics(table);
- this.skipStatsUpdate = StatsSetupConst.STATS_REQUIRE_COMPUTE.stream()
- .anyMatch(providedBasicStats::containsKey);
+ // the handler counts a write's rows into the parameters at commit;
only an ANALYZE scan has a count to apply
+ this.skipStatsUpdate = !work.isExplicitAnalyze();
}
}