szehon-ho commented on code in PR #5376:
URL: https://github.com/apache/iceberg/pull/5376#discussion_r957730131
##########
core/src/main/java/org/apache/iceberg/BaseFilesTable.java:
##########
@@ -186,4 +239,41 @@ ManifestFile manifest() {
return manifest;
}
}
+
+ static class ContentFileStructWithMetrics implements StructLike {
+ private final int structSize;
+ private final StructLike fileAsStruct;
+ private final Map<String, StructLike> readableMetrics;
+
+ ContentFileStructWithMetrics(
+ int structSize, StructLike fileAsStruct, Map<String, StructLike>
readableMetrics) {
+ this.structSize = structSize;
+ this.fileAsStruct = fileAsStruct;
+ this.readableMetrics = readableMetrics;
+ }
+
+ @Override
+ public int size() {
+ return structSize;
+ }
+
+ @Override
+ public <T> T get(int pos, Class<T> javaClass) {
+ if (pos < (structSize - 1)) {
+ return fileAsStruct.get(pos, javaClass);
+ } else if (pos == (structSize - 1)) {
+ return javaClass.cast(readableMetrics);
+ } else {
+ throw new IllegalArgumentException(
+ String.format(
+ "Illegal position access for ContentFileStructWithMetrics: %d,
max allowed is %d",
+ pos, (structSize - 1)));
+ }
+ }
+
+ @Override
+ public <T> void set(int pos, T value) {
+ throw new UnsupportedOperationException("FileEntryRow is read only");
Review Comment:
Sorry thought I updated, should be updated now
##########
core/src/main/java/org/apache/iceberg/BaseFilesTable.java:
##########
@@ -140,42 +143,92 @@ protected CloseableIterable<FileScanTask> doPlanFiles() {
}
static class ManifestReadTask extends BaseFileScanTask implements DataTask {
+
+ static final Set<Integer> READABLE_METRICS_FIELD_IDS =
+ TypeUtil.getProjectedIds(DataFile.READABLE_METRICS.type());
+ static final Schema MIN_PROJECTION_FOR_READABLE_METRICS =
+ new Schema(
+ DataFile.COLUMN_SIZES,
+ DataFile.VALUE_COUNTS,
+ DataFile.NULL_VALUE_COUNTS,
+ DataFile.NAN_VALUE_COUNTS,
+ DataFile.LOWER_BOUNDS,
+ DataFile.UPPER_BOUNDS);
+
private final FileIO io;
private final Map<Integer, PartitionSpec> specsById;
private final ManifestFile manifest;
- private final Schema schema;
+ private final Schema dataTableSchema;
+ private final Schema projection;
ManifestReadTask(
Table table,
ManifestFile manifest,
- Schema schema,
+ Schema projection,
String schemaString,
String specString,
ResidualEvaluator residuals) {
super(DataFiles.fromManifest(manifest), null, schemaString, specString,
residuals);
this.io = table.io();
this.specsById = Maps.newHashMap(table.specs());
this.manifest = manifest;
- this.schema = schema;
+ this.dataTableSchema = table.schema();
+ this.projection = projection;
}
@Override
public CloseableIterable<StructLike> rows() {
- return CloseableIterable.transform(manifestEntries(), file ->
(StructLike) file);
+ if (projection.findColumnName(DataFile.READABLE_METRICS.fieldId()) ==
null) {
+ return CloseableIterable.transform(files(projection), file ->
(StructLike) file);
+ } else {
+ Schema fileProjection = TypeUtil.selectNot(projection,
READABLE_METRICS_FIELD_IDS);
+ Schema minProjection =
+ TypeUtil.joinCommon(fileProjection,
MIN_PROJECTION_FOR_READABLE_METRICS);
+ return CloseableIterable.transform(files(minProjection),
this::withReadableMetrics);
+ }
}
- private CloseableIterable<? extends ContentFile<?>> manifestEntries() {
+ private CloseableIterable<? extends ContentFile<?>> files(Schema
fileProjection) {
switch (manifest.content()) {
case DATA:
- return ManifestFiles.read(manifest, io, specsById).project(schema);
+ return ManifestFiles.read(manifest, io,
specsById).project(fileProjection);
case DELETES:
- return ManifestFiles.readDeleteManifest(manifest, io,
specsById).project(schema);
+ return ManifestFiles.readDeleteManifest(manifest, io,
specsById).project(fileProjection);
default:
throw new IllegalArgumentException(
"Unsupported manifest content type:" + manifest.content());
}
}
+ private StructLike withReadableMetrics(ContentFile<?> file) {
+ int structSize = projection.columns().size();
+ Map<String, StructLike> metrics =
+ MetricsUtil.readableMetricsMap(
+ dataTableSchema, file, readableMetricsProjection(projection));
+ return new ContentFileStructWithMetrics(structSize, (StructLike) file,
metrics);
+ }
+
+ // Handles projections for readable metrics struct
+ private Map<Integer, Integer> readableMetricsProjection(Schema
projectedSchema) {
Review Comment:
Good idea, moved.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
