This is an automated email from the ASF dual-hosted git repository. sorabh pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/drill.git
commit 108ced0ea10dc379f2b60b32698b6aa26683b585 Author: Vitalii Diravka <[email protected]> AuthorDate: Wed Apr 17 15:46:54 2019 +0300 DRILL-7098: File Metadata Metastore Plugin closes #1754 --- exec/java-exec/pom.xml | 10 +++ .../org/apache/drill/exec/expr/IsPredicate.java | 4 +- .../base/AbstractGroupScanWithMetadata.java | 13 ++-- .../apache/drill/exec/physical/base/GroupScan.java | 2 - .../base/SimpleFileTableMetadataProvider.java | 2 +- .../impl/statistics/AbstractMergedStatistic.java | 2 +- .../logical/ConvertCountToDirectScanRule.java | 4 +- .../physical/ConvertCountToDirectScanPrule.java | 3 +- .../exec/record/metadata/MapColumnMetadata.java | 3 +- .../drill/exec/record/metadata/MetadataUtils.java | 37 +++++++++++ .../store/parquet/AbstractParquetGroupScan.java | 3 +- .../store/parquet/BaseParquetMetadataProvider.java | 11 ++-- .../store/parquet/ParquetGroupScanStatistics.java | 8 +-- .../store/parquet/ParquetTableMetadataUtils.java | 10 +-- .../store/parquet/metadata/MetadataPathUtils.java | 13 ++-- exec/jdbc-all/pom.xml | 4 +- .../org/apache/drill/jdbc/impl/DrillMetaImpl.java | 17 ++--- metastore/file-metadata/pom.xml | 41 ++++++++++++ .../exec/physical/base/TableMetadataProvider.java | 0 .../base/TableMetadataProviderBuilder.java | 0 .../org/apache/drill/metastore/FileMetadata.java | 0 .../apache/drill/metastore/FileTableMetadata.java | 0 metastore/metastore-api/pom.xml | 33 ++++++++++ .../drill/exec/expr/ExactStatisticsConstants.java | 0 .../exec/physical/impl/statistics/Statistic.java | 36 ++++++----- .../exec/record/metadata/SchemaPathUtils.java | 34 ---------- .../org/apache/drill/metastore/BaseMetadata.java | 0 .../metastore/CollectableColumnStatisticsKind.java | 0 .../metastore/CollectableTableStatisticsKind.java | 0 .../apache/drill/metastore/ColumnStatistics.java | 0 .../drill/metastore/ColumnStatisticsImpl.java | 0 .../drill/metastore/ColumnStatisticsKind.java | 5 +- .../apache/drill/metastore/LocationProvider.java | 0 .../apache/drill/metastore/PartitionMetadata.java | 0 .../apache/drill/metastore/RowGroupMetadata.java | 0 .../org/apache/drill/metastore/StatisticsKind.java | 0 .../org/apache/drill/metastore/TableMetadata.java | 0 .../drill/metastore/TableStatisticsKind.java | 7 +- metastore/pom.xml | 75 ++++++++++++++++++++++ pom.xml | 1 + 40 files changed, 268 insertions(+), 110 deletions(-) diff --git a/exec/java-exec/pom.xml b/exec/java-exec/pom.xml index c35e935..366b174 100644 --- a/exec/java-exec/pom.xml +++ b/exec/java-exec/pom.xml @@ -287,6 +287,16 @@ <version>${project.version}</version> </dependency> <dependency> + <groupId>org.apache.drill.metastore</groupId> + <artifactId>drill-metastore-api</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>org.apache.drill.metastore</groupId> + <artifactId>drill-file-metastore-plugin</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> <groupId>com.beust</groupId> <artifactId>jcommander</artifactId> <version>1.30</version> diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/IsPredicate.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/IsPredicate.java index c30879d..37a8d0d 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/IsPredicate.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/IsPredicate.java @@ -18,7 +18,7 @@ package org.apache.drill.exec.expr; import org.apache.drill.exec.expr.stat.RowsMatch; -import org.apache.drill.exec.physical.base.GroupScan; +import org.apache.drill.exec.physical.impl.statistics.Statistic; import org.apache.drill.metastore.ColumnStatistics; import org.apache.drill.metastore.ColumnStatisticsKind; import org.apache.drill.shaded.guava.com.google.common.base.Preconditions; @@ -76,7 +76,7 @@ public class IsPredicate<C extends Comparable<C>> extends LogicalExpressionBase || !stat.containsStatistic(ColumnStatisticsKind.MIN_VALUE) || !stat.containsStatistic(ColumnStatisticsKind.MAX_VALUE) || !stat.containsStatistic(ColumnStatisticsKind.NULLS_COUNT) - || (long) stat.getStatistic(ColumnStatisticsKind.NULLS_COUNT) == GroupScan.NO_COLUMN_STATS; + || (long) stat.getStatistic(ColumnStatisticsKind.NULLS_COUNT) == Statistic.NO_COLUMN_STATS; } /** diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/AbstractGroupScanWithMetadata.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/AbstractGroupScanWithMetadata.java index 15e1387..a547fb8 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/AbstractGroupScanWithMetadata.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/AbstractGroupScanWithMetadata.java @@ -34,6 +34,7 @@ import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry; import org.apache.drill.exec.expr.stat.RowsMatch; import org.apache.drill.exec.ops.OptimizerRulesContext; import org.apache.drill.exec.ops.UdfUtilities; +import org.apache.drill.exec.physical.impl.statistics.Statistic; import org.apache.drill.exec.planner.physical.PlannerSettings; import org.apache.drill.exec.record.MaterializedField; import org.apache.drill.exec.record.metadata.ColumnMetadata; @@ -155,13 +156,13 @@ public abstract class AbstractGroupScanWithMetadata extends AbstractFileGroupSca long colNulls; if (columnStats != null) { Long nulls = (Long) columnStats.getStatistic(ColumnStatisticsKind.NULLS_COUNT); - colNulls = nulls != null ? nulls : GroupScan.NO_COLUMN_STATS; + colNulls = nulls != null ? nulls : Statistic.NO_COLUMN_STATS; } else { return 0; } - return GroupScan.NO_COLUMN_STATS == tableRowCount - || GroupScan.NO_COLUMN_STATS == colNulls - ? GroupScan.NO_COLUMN_STATS : tableRowCount - colNulls; + return Statistic.NO_COLUMN_STATS == tableRowCount + || Statistic.NO_COLUMN_STATS == colNulls + ? Statistic.NO_COLUMN_STATS : tableRowCount - colNulls; } @Override @@ -363,7 +364,7 @@ public abstract class AbstractGroupScanWithMetadata extends AbstractFileGroupSca GroupScanWithMetadataFilterer prunedMetadata = getFilterer(); if (getTableMetadata() != null) { long tableRowCount = (long) TableStatisticsKind.ROW_COUNT.getValue(getTableMetadata()); - if (tableRowCount == NO_COLUMN_STATS || tableRowCount <= maxRecords) { + if (tableRowCount == Statistic.NO_COLUMN_STATS || tableRowCount <= maxRecords) { logger.debug("limit push down does not apply, since total number of rows [{}] is less or equal to the required [{}].", tableRowCount, maxRecords); return null; @@ -428,7 +429,7 @@ public abstract class AbstractGroupScanWithMetadata extends AbstractFileGroupSca int currentRowCount = 0; for (T metadata : metadataList) { long rowCount = (long) TableStatisticsKind.ROW_COUNT.getValue(metadata); - if (rowCount == NO_COLUMN_STATS) { + if (rowCount == Statistic.NO_COLUMN_STATS) { return null; } else if (currentRowCount + rowCount <= maxRecords) { currentRowCount += rowCount; diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/GroupScan.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/GroupScan.java index ce74da1..cd49a1e 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/GroupScan.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/GroupScan.java @@ -46,8 +46,6 @@ public interface GroupScan extends Scan, HasAffinity { */ List<SchemaPath> ALL_COLUMNS = ImmutableList.of(SchemaPath.STAR_COLUMN); - long NO_COLUMN_STATS = -1; - void applyAssignments(List<DrillbitEndpoint> endpoints) throws PhysicalOperatorSetupException; SubScan getSpecificScan(int minorFragmentId) throws ExecutionSetupException; diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/SimpleFileTableMetadataProvider.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/SimpleFileTableMetadataProvider.java index 98c0c19..1efad36 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/SimpleFileTableMetadataProvider.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/SimpleFileTableMetadataProvider.java @@ -124,7 +124,7 @@ public class SimpleFileTableMetadataProvider implements TableMetadataProvider { @Override @SuppressWarnings("unchecked") - public TableMetadataProvider build() throws IOException { + public TableMetadataProvider build() { SchemaProvider schemaProvider = metadataProviderManager.getSchemaProvider(); TableMetadataProvider source = metadataProviderManager.getTableMetadataProvider(); if (source == null) { diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/statistics/AbstractMergedStatistic.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/statistics/AbstractMergedStatistic.java index a4ee74f..4e9a762 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/statistics/AbstractMergedStatistic.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/statistics/AbstractMergedStatistic.java @@ -19,7 +19,7 @@ package org.apache.drill.exec.physical.impl.statistics; import org.apache.drill.exec.vector.complex.MapVector; -public abstract class AbstractMergedStatistic extends Statistic implements MergedStatistic { +public abstract class AbstractMergedStatistic implements MergedStatistic, Statistic { protected String name; protected String inputName; protected double samplePercent; diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/ConvertCountToDirectScanRule.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/ConvertCountToDirectScanRule.java index fd84e20..7375499 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/ConvertCountToDirectScanRule.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/ConvertCountToDirectScanRule.java @@ -31,8 +31,8 @@ import org.apache.commons.lang3.tuple.Pair; import org.apache.drill.common.expression.SchemaPath; import org.apache.drill.common.logical.FormatPluginConfig; -import org.apache.drill.exec.physical.base.GroupScan; import org.apache.drill.exec.physical.base.ScanStats; +import org.apache.drill.exec.physical.impl.statistics.Statistic; import org.apache.drill.exec.planner.common.CountToDirectScanUtils; import org.apache.drill.exec.planner.common.DrillRelOptUtil; @@ -288,7 +288,7 @@ public class ConvertCountToDirectScanRule extends RelOptRule { Metadata_V4.ColumnTypeMetadata_v4 columnMetadata = metadataSummary.getColumnTypeInfo(new Metadata_V4.ColumnTypeMetadata_v4.Key(simplePath)); - if (columnMetadata == null || columnMetadata.totalNullCount == GroupScan.NO_COLUMN_STATS) { + if (columnMetadata == null || columnMetadata.totalNullCount == Statistic.NO_COLUMN_STATS) { // if column stats is not available don't apply this rule, return empty counts return ImmutableMap.of(); } else { diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ConvertCountToDirectScanPrule.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ConvertCountToDirectScanPrule.java index 4176950..8d7022e 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ConvertCountToDirectScanPrule.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ConvertCountToDirectScanPrule.java @@ -33,6 +33,7 @@ import org.apache.calcite.rel.core.AggregateCall; import org.apache.drill.common.expression.SchemaPath; import org.apache.drill.exec.physical.base.GroupScan; import org.apache.drill.exec.physical.base.ScanStats; +import org.apache.drill.exec.physical.impl.statistics.Statistic; import org.apache.drill.exec.planner.logical.DrillAggregateRel; import org.apache.drill.exec.planner.logical.DrillProjectRel; import org.apache.drill.exec.planner.logical.DrillScanRel; @@ -202,7 +203,7 @@ public class ConvertCountToDirectScanPrule extends Prule { } cnt = oldGrpScan.getColumnValueCount(simplePath); - if (cnt == GroupScan.NO_COLUMN_STATS) { + if (cnt == Statistic.NO_COLUMN_STATS) { // if column stats is not available don't apply this rule, return empty counts return ImmutableMap.of(); } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/record/metadata/MapColumnMetadata.java b/exec/java-exec/src/main/java/org/apache/drill/exec/record/metadata/MapColumnMetadata.java index bf1f9af..f9c05f4 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/record/metadata/MapColumnMetadata.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/record/metadata/MapColumnMetadata.java @@ -64,8 +64,7 @@ public class MapColumnMetadata extends AbstractColumnMetadata { mapSchema = (TupleSchema) from.mapSchema.copy(); } - public MapColumnMetadata(String name, DataMode mode, - TupleSchema mapSchema) { + public MapColumnMetadata(String name, DataMode mode, TupleSchema mapSchema) { super(name, MinorType.MAP, mode); if (mapSchema == null) { this.mapSchema = new TupleSchema(); diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/record/metadata/MetadataUtils.java b/exec/java-exec/src/main/java/org/apache/drill/exec/record/metadata/MetadataUtils.java index 21577c0..f1cf45a 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/record/metadata/MetadataUtils.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/record/metadata/MetadataUtils.java @@ -19,6 +19,10 @@ package org.apache.drill.exec.record.metadata; import java.util.List; +import org.apache.drill.common.exceptions.DrillRuntimeException; +import org.apache.drill.common.expression.PathSegment; +import org.apache.drill.common.expression.SchemaPath; +import org.apache.drill.common.types.TypeProtos; import org.apache.drill.common.types.TypeProtos.DataMode; import org.apache.drill.common.types.TypeProtos.MajorType; import org.apache.drill.common.types.TypeProtos.MinorType; @@ -203,4 +207,37 @@ public class MetadataUtils { .build(); return new PrimitiveColumnMetadata(field); } + + /** + * Adds column with specified schema path and type into specified {@code TupleMetadata schema}. + * + * @param schema tuple schema where column should be added + * @param schemaPath schema path of the column which should be added + * @param type type of the column which should be added + */ + public static void addColumnMetadata(TupleMetadata schema, SchemaPath schemaPath, TypeProtos.MajorType type) { + PathSegment.NameSegment colPath = schemaPath.getUnIndexed().getRootSegment(); + ColumnMetadata colMetadata; + + while (!colPath.isLastPath()) { + colMetadata = schema.metadata(colPath.getPath()); + if (colMetadata == null) { + colMetadata = MetadataUtils.newMap(colPath.getPath(), null); + schema.addColumn(colMetadata); + } + if (!colMetadata.isMap()) { + throw new DrillRuntimeException(String.format("Expected map, but was %s", colMetadata.majorType())); + } + + schema = colMetadata.mapSchema(); + colPath = (PathSegment.NameSegment) colPath.getChild(); + } + + colMetadata = schema.metadata(colPath.getPath()); + if (colMetadata == null) { + schema.addColumn(new PrimitiveColumnMetadata(MaterializedField.create(colPath.getPath(), type))); + } else if (!colMetadata.majorType().equals(type)) { + throw new DrillRuntimeException(String.format("Types mismatch: existing type: %s, new type: %s", colMetadata.majorType(), type)); + } + } } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/AbstractParquetGroupScan.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/AbstractParquetGroupScan.java index e368bb3..34cf354 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/AbstractParquetGroupScan.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/AbstractParquetGroupScan.java @@ -24,6 +24,7 @@ import org.apache.commons.collections.CollectionUtils; import org.apache.drill.common.expression.ExpressionStringBuilder; import org.apache.drill.exec.physical.base.AbstractGroupScanWithMetadata; import org.apache.drill.exec.physical.base.ParquetMetadataProvider; +import org.apache.drill.exec.physical.impl.statistics.Statistic; import org.apache.drill.metastore.BaseMetadata; import org.apache.drill.metastore.LocationProvider; import org.apache.drill.metastore.PartitionMetadata; @@ -330,7 +331,7 @@ public abstract class AbstractParquetGroupScan extends AbstractGroupScanWithMeta maxRecords = Math.max(maxRecords, 1); // Make sure it request at least 1 row -> 1 rowGroup. if (getTableMetadata() != null) { long tableRowCount = (long) TableStatisticsKind.ROW_COUNT.getValue(getTableMetadata()); - if (tableRowCount == NO_COLUMN_STATS || tableRowCount <= maxRecords) { + if (tableRowCount == Statistic.NO_COLUMN_STATS || tableRowCount <= maxRecords) { logger.debug("limit push down does not apply, since total number of rows [{}] is less or equal to the required [{}].", tableRowCount, maxRecords); return null; diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/BaseParquetMetadataProvider.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/BaseParquetMetadataProvider.java index 29126b3..1877356 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/BaseParquetMetadataProvider.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/BaseParquetMetadataProvider.java @@ -17,9 +17,10 @@ */ package org.apache.drill.exec.store.parquet; -import org.apache.drill.exec.physical.base.GroupScan; import org.apache.drill.exec.physical.base.ParquetMetadataProvider; +import org.apache.drill.exec.physical.impl.statistics.Statistic; import org.apache.drill.exec.planner.common.DrillStatsTable; +import org.apache.drill.exec.record.metadata.MetadataUtils; import org.apache.drill.exec.record.metadata.TupleMetadata; import org.apache.drill.metastore.BaseMetadata; import org.apache.drill.metastore.ColumnStatisticsImpl; @@ -190,12 +191,12 @@ public abstract class BaseParquetMetadataProvider implements ParquetMetadataProv if (this.schema == null) { schema = new TupleSchema(); - fields.forEach((schemaPath, majorType) -> SchemaPathUtils.addColumnMetadata(schema, schemaPath, majorType)); + fields.forEach((schemaPath, majorType) -> MetadataUtils.addColumnMetadata(schema, schemaPath, majorType)); } else { // merges specified schema with schema from table fields.forEach((schemaPath, majorType) -> { if (SchemaPathUtils.getColumnMetadata(schemaPath, schema) == null) { - SchemaPathUtils.addColumnMetadata(schema, schemaPath, majorType); + MetadataUtils.addColumnMetadata(schema, schemaPath, majorType); } }); } @@ -306,8 +307,8 @@ public abstract class BaseParquetMetadataProvider implements ParquetMetadataProv statistics.put(ColumnStatisticsKind.MIN_VALUE, partitionKey); statistics.put(ColumnStatisticsKind.MAX_VALUE, partitionKey); - statistics.put(ColumnStatisticsKind.NULLS_COUNT, GroupScan.NO_COLUMN_STATS); - statistics.put(TableStatisticsKind.ROW_COUNT, GroupScan.NO_COLUMN_STATS); + statistics.put(ColumnStatisticsKind.NULLS_COUNT, Statistic.NO_COLUMN_STATS); + statistics.put(TableStatisticsKind.ROW_COUNT, Statistic.NO_COLUMN_STATS); columnsStatistics.put(partitionColumn, new ColumnStatisticsImpl<>(statistics, ParquetTableMetadataUtils.getComparator(getParquetGroupScanStatistics().getTypeForColumn(partitionColumn).getMinorType()))); diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetGroupScanStatistics.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetGroupScanStatistics.java index d53c038..c53cee9 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetGroupScanStatistics.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetGroupScanStatistics.java @@ -20,7 +20,7 @@ package org.apache.drill.exec.store.parquet; import org.apache.commons.lang3.mutable.MutableLong; import org.apache.drill.common.expression.SchemaPath; import org.apache.drill.common.types.TypeProtos; -import org.apache.drill.exec.physical.base.GroupScan; +import org.apache.drill.exec.physical.impl.statistics.Statistic; import org.apache.drill.exec.record.metadata.ColumnMetadata; import org.apache.drill.exec.record.metadata.SchemaPathUtils; import org.apache.drill.metastore.BaseMetadata; @@ -109,10 +109,10 @@ public class ParquetGroupScanStatistics<T extends BaseMetadata & LocationProvide previousCount = emptyCount; } Long nullsNum = (Long) statistics.getStatistic(ColumnStatisticsKind.NULLS_COUNT); - if (previousCount.longValue() != GroupScan.NO_COLUMN_STATS && nullsNum != null && nullsNum != GroupScan.NO_COLUMN_STATS) { + if (previousCount.longValue() != Statistic.NO_COLUMN_STATS && nullsNum != null && nullsNum != Statistic.NO_COLUMN_STATS) { previousCount.add(localRowCount - nullsNum); } else { - previousCount.setValue(GroupScan.NO_COLUMN_STATS); + previousCount.setValue(Statistic.NO_COLUMN_STATS); } ColumnMetadata columnMetadata = SchemaPathUtils.getColumnMetadata(schemaPath, metadata.getSchema()); TypeProtos.MajorType majorType = columnMetadata != null ? columnMetadata.majorType() : null; @@ -207,7 +207,7 @@ public class ParquetGroupScanStatistics<T extends BaseMetadata & LocationProvide private boolean isSingleVal(ColumnStatistics columnStatistics, long rowCount) { Long numNulls = (Long) columnStatistics.getStatistic(ColumnStatisticsKind.NULLS_COUNT); - if (numNulls != null && numNulls != GroupScan.NO_COLUMN_STATS) { + if (numNulls != null && numNulls != Statistic.NO_COLUMN_STATS) { Object min = columnStatistics.getStatistic(ColumnStatisticsKind.MIN_VALUE); Object max = columnStatistics.getStatistic(ColumnStatisticsKind.MAX_VALUE); if (min != null) { diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetTableMetadataUtils.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetTableMetadataUtils.java index b8a912d..8df6585 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetTableMetadataUtils.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetTableMetadataUtils.java @@ -19,8 +19,8 @@ package org.apache.drill.exec.store.parquet; import org.apache.drill.common.expression.SchemaPath; import org.apache.drill.common.types.TypeProtos; -import org.apache.drill.exec.physical.base.GroupScan; -import org.apache.drill.exec.record.metadata.SchemaPathUtils; +import org.apache.drill.exec.physical.impl.statistics.Statistic; +import org.apache.drill.exec.record.metadata.MetadataUtils; import org.apache.drill.exec.record.metadata.TupleMetadata; import org.apache.drill.exec.record.metadata.TupleSchema; import org.apache.drill.exec.resolver.TypeCastRules; @@ -155,7 +155,7 @@ public class ParquetTableMetadataUtils { Map<SchemaPath, TypeProtos.MajorType> columns = getRowGroupFields(tableMetadata, rowGroupMetadata); TupleSchema schema = new TupleSchema(); - columns.forEach((schemaPath, majorType) -> SchemaPathUtils.addColumnMetadata(schema, schemaPath, majorType)); + columns.forEach((schemaPath, majorType) -> MetadataUtils.addColumnMetadata(schema, schemaPath, majorType)); return new RowGroupMetadata( schema, columnsStatistics, rowGroupStatistics, rowGroupMetadata.getHostAffinity(), rgIndexInFile, location); @@ -275,7 +275,7 @@ public class ParquetTableMetadataUtils { Long nulls = column.getNulls(); if (!column.isNumNullsSet() || nulls == null) { - nulls = GroupScan.NO_COLUMN_STATS; + nulls = Statistic.NO_COLUMN_STATS; } PrimitiveType.PrimitiveTypeName primitiveType = getPrimitiveTypeName(tableMetadata, column); OriginalType originalType = getOriginalType(tableMetadata, column); @@ -309,7 +309,7 @@ public class ParquetTableMetadataUtils { SchemaPath schemaPath = SchemaPath.getCompoundPath(columnTypeMetadata.name); if (!schemaPaths.contains(schemaPath)) { Map<StatisticsKind, Object> statistics = new HashMap<>(); - statistics.put(ColumnStatisticsKind.NULLS_COUNT, GroupScan.NO_COLUMN_STATS); + statistics.put(ColumnStatisticsKind.NULLS_COUNT, Statistic.NO_COLUMN_STATS); PrimitiveType.PrimitiveTypeName primitiveType = columnTypeMetadata.primitiveType; OriginalType originalType = columnTypeMetadata.originalType; Comparator comparator = getComparator(primitiveType, originalType); diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/metadata/MetadataPathUtils.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/metadata/MetadataPathUtils.java index b94f135..2cad8e1 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/metadata/MetadataPathUtils.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/metadata/MetadataPathUtils.java @@ -17,7 +17,6 @@ */ package org.apache.drill.exec.store.parquet.metadata; -import org.apache.drill.shaded.guava.com.google.common.collect.Lists; import org.apache.drill.common.util.DrillVersionInfo; import org.apache.hadoop.fs.Path; @@ -44,10 +43,9 @@ public class MetadataPathUtils { */ public static List<Path> convertToAbsolutePaths(List<Path> paths, String baseDir) { if (!paths.isEmpty()) { - List<Path> absolutePaths = Lists.newArrayList(); + List<Path> absolutePaths = new ArrayList<>(); for (Path relativePath : paths) { - Path absolutePath = (relativePath.isAbsolute()) ? relativePath - : new Path(baseDir, relativePath); + Path absolutePath = (relativePath.isAbsolute()) ? relativePath : new Path(baseDir, relativePath); absolutePaths.add(absolutePath); } return absolutePaths; @@ -62,9 +60,10 @@ public class MetadataPathUtils { * @param baseDir base parent directory * @return list of files with absolute paths */ - public static List<? extends ParquetFileMetadata> convertToFilesWithAbsolutePaths(List<? extends ParquetFileMetadata> files, String baseDir) { + public static List<? extends ParquetFileMetadata> convertToFilesWithAbsolutePaths( + List<? extends ParquetFileMetadata> files, String baseDir) { if (!files.isEmpty()) { - List<ParquetFileMetadata> filesWithAbsolutePaths = Lists.newArrayList(); + List<ParquetFileMetadata> filesWithAbsolutePaths = new ArrayList<>(); for (ParquetFileMetadata file : files) { Path relativePath = file.getPath(); ParquetFileMetadata fileWithAbsolutePath = null; @@ -97,7 +96,7 @@ public class MetadataPathUtils { for (Path directory : tableMetadataWithAbsolutePaths.getDirectories()) { directoriesWithRelativePaths.add(relativize(baseDir, directory)); } - List<ParquetFileMetadata_v4> filesWithRelativePaths = Lists.newArrayList(); + List<ParquetFileMetadata_v4> filesWithRelativePaths = new ArrayList<>(); for (ParquetFileMetadata_v4 file : (List<ParquetFileMetadata_v4>) tableMetadataWithAbsolutePaths.getFiles()) { filesWithRelativePaths.add(new ParquetFileMetadata_v4( relativize(baseDir, file.getPath()), file.length, file.rowGroups)); diff --git a/exec/jdbc-all/pom.xml b/exec/jdbc-all/pom.xml index 4367489..be79b41 100644 --- a/exec/jdbc-all/pom.xml +++ b/exec/jdbc-all/pom.xml @@ -347,7 +347,6 @@ <!-- Relocate Drill classes to minimize classloader hell. --> <relocation><pattern>org.apache.drill.exec.</pattern><shadedPattern>oadd.org.apache.drill.exec.</shadedPattern></relocation> <relocation><pattern>org.apache.drill.common.</pattern><shadedPattern>oadd.org.apache.drill.common.</shadedPattern></relocation> - <relocation><pattern>org.apache.drill.metastore.</pattern><shadedPattern>oadd.org.apache.drill.metastore.</shadedPattern></relocation> <!-- Move dependencies out of path --> <relocation><pattern>antlr.</pattern><shadedPattern>oadd.antlr.</shadedPattern></relocation> @@ -489,6 +488,7 @@ <exclude>org/apache/commons/pool2/**</exclude> <exclude>org/apache/http/**</exclude> <exclude>org/apache/directory/**</exclude> + <exclude>org/apache/drill/metastore/**</exclude> <exclude>com/jcraft/**</exclude> <exclude>**/mapr/**</exclude> <exclude>org/yaml/**</exclude> @@ -664,7 +664,6 @@ <!-- Relocate Drill classes to minimize classloader hell. --> <relocation><pattern>org.apache.drill.exec.</pattern><shadedPattern>oadd.org.apache.drill.exec.</shadedPattern></relocation> <relocation><pattern>org.apache.drill.common.</pattern><shadedPattern>oadd.org.apache.drill.common.</shadedPattern></relocation> - <relocation><pattern>org.apache.drill.metastore.</pattern><shadedPattern>oadd.org.apache.drill.metastore.</shadedPattern></relocation> <!-- Move dependencies out of path --> <relocation><pattern>antlr.</pattern><shadedPattern>oadd.antlr.</shadedPattern></relocation> @@ -788,6 +787,7 @@ <exclude>org/apache/drill/exec/rpc/data/**</exclude> <exclude>org/apache/drill/exec/rpc/control/**</exclude> <exclude>org/apache/drill/exec/work/**</exclude> + <exclude>org/apache/drill/metastore/**</exclude> <exclude>org/apache/hadoop/**</exclude> <exclude>org/apache/commons/pool2/**</exclude> <exclude>org/apache/http/**</exclude> diff --git a/exec/jdbc/src/main/java/org/apache/drill/jdbc/impl/DrillMetaImpl.java b/exec/jdbc/src/main/java/org/apache/drill/jdbc/impl/DrillMetaImpl.java index 2b61a6a..e06445d 100644 --- a/exec/jdbc/src/main/java/org/apache/drill/jdbc/impl/DrillMetaImpl.java +++ b/exec/jdbc/src/main/java/org/apache/drill/jdbc/impl/DrillMetaImpl.java @@ -29,6 +29,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; import javax.validation.constraints.NotNull; @@ -60,9 +61,7 @@ import org.apache.drill.exec.proto.UserProtos.TableMetadata; import org.apache.drill.exec.rpc.DrillRpcFuture; import org.apache.drill.exec.rpc.RpcException; -import org.apache.drill.shaded.guava.com.google.common.base.Function; import org.apache.drill.shaded.guava.com.google.common.collect.ImmutableMap; -import org.apache.drill.shaded.guava.com.google.common.collect.Lists; public class DrillMetaImpl extends MetaImpl { private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillMetaImpl.class); @@ -315,18 +314,14 @@ public class DrillMetaImpl extends MetaImpl { } try { - List<Object> tables = Lists.transform(getResult(response), new Function<ResponseValue, Object>() { - @Override - public Object apply(ResponseValue input) { - return adapt(input); - } - }); + List<Object> tables = getResult(response).stream() + .map(this::adapt) + .collect(Collectors.toList()); Meta.Frame frame = Meta.Frame.create(0, true, tables); StructType fieldMetaData = drillFieldMetaData(clazz); - Meta.Signature signature = Meta.Signature.create( - fieldMetaData.columns, "", - Collections.<AvaticaParameter>emptyList(), CursorFactory.record(clazz), Meta.StatementType.SELECT); + Meta.Signature signature = Meta.Signature.create(fieldMetaData.columns, "", Collections.emptyList(), + CursorFactory.record(clazz), Meta.StatementType.SELECT); AvaticaStatement statement = connection.createStatement(); return MetaResultSet.create(connection.id, statement.getId(), true, diff --git a/metastore/file-metadata/pom.xml b/metastore/file-metadata/pom.xml new file mode 100644 index 0000000..d0bd794 --- /dev/null +++ b/metastore/file-metadata/pom.xml @@ -0,0 +1,41 @@ +<?xml version="1.0"?> +<!-- + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +--> +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + + <parent> + <groupId>org.apache.drill.metastore</groupId> + <artifactId>metastore-parent</artifactId> + <version>1.17.0-SNAPSHOT</version> + </parent> + + <artifactId>drill-file-metastore-plugin</artifactId> + <name>Drill File Metadata Metastore Plugin</name> + + <dependencies> + <dependency> + <groupId>org.apache.drill.metastore</groupId> + <artifactId>drill-metastore-api</artifactId> + <version>${project.version}</version> + </dependency> + </dependencies> + +</project> diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/TableMetadataProvider.java b/metastore/file-metadata/src/main/java/org/apache/drill/exec/physical/base/TableMetadataProvider.java similarity index 100% rename from exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/TableMetadataProvider.java rename to metastore/file-metadata/src/main/java/org/apache/drill/exec/physical/base/TableMetadataProvider.java diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/TableMetadataProviderBuilder.java b/metastore/file-metadata/src/main/java/org/apache/drill/exec/physical/base/TableMetadataProviderBuilder.java similarity index 100% rename from exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/TableMetadataProviderBuilder.java rename to metastore/file-metadata/src/main/java/org/apache/drill/exec/physical/base/TableMetadataProviderBuilder.java diff --git a/exec/java-exec/src/main/java/org/apache/drill/metastore/FileMetadata.java b/metastore/file-metadata/src/main/java/org/apache/drill/metastore/FileMetadata.java similarity index 100% rename from exec/java-exec/src/main/java/org/apache/drill/metastore/FileMetadata.java rename to metastore/file-metadata/src/main/java/org/apache/drill/metastore/FileMetadata.java diff --git a/exec/java-exec/src/main/java/org/apache/drill/metastore/FileTableMetadata.java b/metastore/file-metadata/src/main/java/org/apache/drill/metastore/FileTableMetadata.java similarity index 100% rename from exec/java-exec/src/main/java/org/apache/drill/metastore/FileTableMetadata.java rename to metastore/file-metadata/src/main/java/org/apache/drill/metastore/FileTableMetadata.java diff --git a/metastore/metastore-api/pom.xml b/metastore/metastore-api/pom.xml new file mode 100644 index 0000000..d3f3aed --- /dev/null +++ b/metastore/metastore-api/pom.xml @@ -0,0 +1,33 @@ +<?xml version="1.0"?> +<!-- + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +--> +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + + <parent> + <groupId>org.apache.drill.metastore</groupId> + <artifactId>metastore-parent</artifactId> + <version>1.17.0-SNAPSHOT</version> + </parent> + + <artifactId>drill-metastore-api</artifactId> + <name>Drill Metastore API</name> + +</project> diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExactStatisticsConstants.java b/metastore/metastore-api/src/main/java/org/apache/drill/exec/expr/ExactStatisticsConstants.java similarity index 100% rename from exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExactStatisticsConstants.java rename to metastore/metastore-api/src/main/java/org/apache/drill/exec/expr/ExactStatisticsConstants.java diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/statistics/Statistic.java b/metastore/metastore-api/src/main/java/org/apache/drill/exec/physical/impl/statistics/Statistic.java similarity index 54% rename from exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/statistics/Statistic.java rename to metastore/metastore-api/src/main/java/org/apache/drill/exec/physical/impl/statistics/Statistic.java index 8f5392a..5794a13 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/statistics/Statistic.java +++ b/metastore/metastore-api/src/main/java/org/apache/drill/exec/physical/impl/statistics/Statistic.java @@ -20,27 +20,29 @@ package org.apache.drill.exec.physical.impl.statistics; /* * Base Statistics class - all statistics classes should extend this class */ -public abstract class Statistic { +public interface Statistic { /* * The lifecycle states for statistics */ - public enum State {INIT, CONFIG, MERGE, COMPLETE}; + enum State {INIT, CONFIG, MERGE, COMPLETE} + + long NO_COLUMN_STATS = -1; /* * List of statistics used in Drill. */ - public static final String COLNAME = "column"; - public static final String COLTYPE = "majortype"; - public static final String SCHEMA = "schema"; - public static final String COMPUTED = "computed"; - public static final String ROWCOUNT = "rowcount"; - public static final String NNROWCOUNT = "nonnullrowcount"; - public static final String NDV = "approx_count_distinct"; - public static final String HLL_MERGE = "hll_merge"; - public static final String HLL = "hll"; - public static final String AVG_WIDTH = "avg_width"; - public static final String SUM_WIDTH = "sum_width"; - public static final String CNT_DUPS = "approx_count_dups"; - public static final String SUM_DUPS = "sum"; - public static final String TDIGEST = "tdigest"; - public static final String TDIGEST_MERGE = "tdigest_merge"; + String COLNAME = "column"; + String COLTYPE = "majortype"; + String SCHEMA = "schema"; + String COMPUTED = "computed"; + String ROWCOUNT = "rowcount"; + String NNROWCOUNT = "nonnullrowcount"; + String NDV = "approx_count_distinct"; + String HLL_MERGE = "hll_merge"; + String HLL = "hll"; + String AVG_WIDTH = "avg_width"; + String SUM_WIDTH = "sum_width"; + String CNT_DUPS = "approx_count_dups"; + String SUM_DUPS = "sum"; + String TDIGEST = "tdigest"; + String TDIGEST_MERGE = "tdigest_merge"; } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/record/metadata/SchemaPathUtils.java b/metastore/metastore-api/src/main/java/org/apache/drill/exec/record/metadata/SchemaPathUtils.java similarity index 55% rename from exec/java-exec/src/main/java/org/apache/drill/exec/record/metadata/SchemaPathUtils.java rename to metastore/metastore-api/src/main/java/org/apache/drill/exec/record/metadata/SchemaPathUtils.java index 085c68c..5c39701 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/record/metadata/SchemaPathUtils.java +++ b/metastore/metastore-api/src/main/java/org/apache/drill/exec/record/metadata/SchemaPathUtils.java @@ -17,11 +17,8 @@ */ package org.apache.drill.exec.record.metadata; -import org.apache.drill.common.exceptions.DrillRuntimeException; import org.apache.drill.common.expression.PathSegment; import org.apache.drill.common.expression.SchemaPath; -import org.apache.drill.common.types.TypeProtos; -import org.apache.drill.exec.record.MaterializedField; public class SchemaPathUtils { @@ -50,36 +47,5 @@ public class SchemaPathUtils { return colMetadata; } - /** - * Adds column with specified schema path and type into specified {@code TupleMetadata schema}. - * - * @param schema tuple schema where column should be added - * @param schemaPath schema path of the column which should be added - * @param type type of the column which should be added - */ - public static void addColumnMetadata(TupleMetadata schema, SchemaPath schemaPath, TypeProtos.MajorType type) { - PathSegment.NameSegment colPath = schemaPath.getUnIndexed().getRootSegment(); - ColumnMetadata colMetadata; - while (!colPath.isLastPath()) { - colMetadata = schema.metadata(colPath.getPath()); - if (colMetadata == null) { - colMetadata = MetadataUtils.newMap(colPath.getPath(), null); - schema.addColumn(colMetadata); - } - if (!colMetadata.isMap()) { - throw new DrillRuntimeException(String.format("Expected map, but was %s", colMetadata.majorType())); - } - - schema = colMetadata.mapSchema(); - colPath = (PathSegment.NameSegment) colPath.getChild(); - } - - colMetadata = schema.metadata(colPath.getPath()); - if (colMetadata == null) { - schema.addColumn(new PrimitiveColumnMetadata(MaterializedField.create(colPath.getPath(), type))); - } else if (!colMetadata.majorType().equals(type)) { - throw new DrillRuntimeException(String.format("Types mismatch: existing type: %s, new type: %s", colMetadata.majorType(), type)); - } - } } diff --git a/exec/java-exec/src/main/java/org/apache/drill/metastore/BaseMetadata.java b/metastore/metastore-api/src/main/java/org/apache/drill/metastore/BaseMetadata.java similarity index 100% rename from exec/java-exec/src/main/java/org/apache/drill/metastore/BaseMetadata.java rename to metastore/metastore-api/src/main/java/org/apache/drill/metastore/BaseMetadata.java diff --git a/exec/java-exec/src/main/java/org/apache/drill/metastore/CollectableColumnStatisticsKind.java b/metastore/metastore-api/src/main/java/org/apache/drill/metastore/CollectableColumnStatisticsKind.java similarity index 100% rename from exec/java-exec/src/main/java/org/apache/drill/metastore/CollectableColumnStatisticsKind.java rename to metastore/metastore-api/src/main/java/org/apache/drill/metastore/CollectableColumnStatisticsKind.java diff --git a/exec/java-exec/src/main/java/org/apache/drill/metastore/CollectableTableStatisticsKind.java b/metastore/metastore-api/src/main/java/org/apache/drill/metastore/CollectableTableStatisticsKind.java similarity index 100% rename from exec/java-exec/src/main/java/org/apache/drill/metastore/CollectableTableStatisticsKind.java rename to metastore/metastore-api/src/main/java/org/apache/drill/metastore/CollectableTableStatisticsKind.java diff --git a/exec/java-exec/src/main/java/org/apache/drill/metastore/ColumnStatistics.java b/metastore/metastore-api/src/main/java/org/apache/drill/metastore/ColumnStatistics.java similarity index 100% rename from exec/java-exec/src/main/java/org/apache/drill/metastore/ColumnStatistics.java rename to metastore/metastore-api/src/main/java/org/apache/drill/metastore/ColumnStatistics.java diff --git a/exec/java-exec/src/main/java/org/apache/drill/metastore/ColumnStatisticsImpl.java b/metastore/metastore-api/src/main/java/org/apache/drill/metastore/ColumnStatisticsImpl.java similarity index 100% rename from exec/java-exec/src/main/java/org/apache/drill/metastore/ColumnStatisticsImpl.java rename to metastore/metastore-api/src/main/java/org/apache/drill/metastore/ColumnStatisticsImpl.java diff --git a/exec/java-exec/src/main/java/org/apache/drill/metastore/ColumnStatisticsKind.java b/metastore/metastore-api/src/main/java/org/apache/drill/metastore/ColumnStatisticsKind.java similarity index 96% rename from exec/java-exec/src/main/java/org/apache/drill/metastore/ColumnStatisticsKind.java rename to metastore/metastore-api/src/main/java/org/apache/drill/metastore/ColumnStatisticsKind.java index e94b842..8ee63ad 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/metastore/ColumnStatisticsKind.java +++ b/metastore/metastore-api/src/main/java/org/apache/drill/metastore/ColumnStatisticsKind.java @@ -17,7 +17,6 @@ */ package org.apache.drill.metastore; -import org.apache.drill.exec.physical.base.GroupScan; import org.apache.drill.exec.expr.ExactStatisticsConstants; import org.apache.drill.exec.physical.impl.statistics.Statistic; @@ -38,8 +37,8 @@ public enum ColumnStatisticsKind implements CollectableColumnStatisticsKind { long nullsCount = 0; for (ColumnStatistics statistics : statisticsList) { Long statNullsCount = (Long) statistics.getStatistic(this); - if (statNullsCount == null || statNullsCount == GroupScan.NO_COLUMN_STATS) { - return GroupScan.NO_COLUMN_STATS; + if (statNullsCount == null || statNullsCount == Statistic.NO_COLUMN_STATS) { + return Statistic.NO_COLUMN_STATS; } else { nullsCount += statNullsCount; } diff --git a/exec/java-exec/src/main/java/org/apache/drill/metastore/LocationProvider.java b/metastore/metastore-api/src/main/java/org/apache/drill/metastore/LocationProvider.java similarity index 100% rename from exec/java-exec/src/main/java/org/apache/drill/metastore/LocationProvider.java rename to metastore/metastore-api/src/main/java/org/apache/drill/metastore/LocationProvider.java diff --git a/exec/java-exec/src/main/java/org/apache/drill/metastore/PartitionMetadata.java b/metastore/metastore-api/src/main/java/org/apache/drill/metastore/PartitionMetadata.java similarity index 100% rename from exec/java-exec/src/main/java/org/apache/drill/metastore/PartitionMetadata.java rename to metastore/metastore-api/src/main/java/org/apache/drill/metastore/PartitionMetadata.java diff --git a/exec/java-exec/src/main/java/org/apache/drill/metastore/RowGroupMetadata.java b/metastore/metastore-api/src/main/java/org/apache/drill/metastore/RowGroupMetadata.java similarity index 100% rename from exec/java-exec/src/main/java/org/apache/drill/metastore/RowGroupMetadata.java rename to metastore/metastore-api/src/main/java/org/apache/drill/metastore/RowGroupMetadata.java diff --git a/exec/java-exec/src/main/java/org/apache/drill/metastore/StatisticsKind.java b/metastore/metastore-api/src/main/java/org/apache/drill/metastore/StatisticsKind.java similarity index 100% rename from exec/java-exec/src/main/java/org/apache/drill/metastore/StatisticsKind.java rename to metastore/metastore-api/src/main/java/org/apache/drill/metastore/StatisticsKind.java diff --git a/exec/java-exec/src/main/java/org/apache/drill/metastore/TableMetadata.java b/metastore/metastore-api/src/main/java/org/apache/drill/metastore/TableMetadata.java similarity index 100% rename from exec/java-exec/src/main/java/org/apache/drill/metastore/TableMetadata.java rename to metastore/metastore-api/src/main/java/org/apache/drill/metastore/TableMetadata.java diff --git a/exec/java-exec/src/main/java/org/apache/drill/metastore/TableStatisticsKind.java b/metastore/metastore-api/src/main/java/org/apache/drill/metastore/TableStatisticsKind.java similarity index 94% rename from exec/java-exec/src/main/java/org/apache/drill/metastore/TableStatisticsKind.java rename to metastore/metastore-api/src/main/java/org/apache/drill/metastore/TableStatisticsKind.java index 63b9243..1c10938 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/metastore/TableStatisticsKind.java +++ b/metastore/metastore-api/src/main/java/org/apache/drill/metastore/TableStatisticsKind.java @@ -17,7 +17,6 @@ */ package org.apache.drill.metastore; -import org.apache.drill.exec.physical.base.GroupScan; import org.apache.drill.exec.expr.ExactStatisticsConstants; import org.apache.drill.exec.physical.impl.statistics.Statistic; @@ -37,8 +36,8 @@ public enum TableStatisticsKind implements CollectableTableStatisticsKind { long rowCount = 0; for (BaseMetadata statistic : statistics) { Long statRowCount = getValue(statistic); - if (statRowCount == null || statRowCount == GroupScan.NO_COLUMN_STATS) { - rowCount = GroupScan.NO_COLUMN_STATS; + if (statRowCount == null || statRowCount == Statistic.NO_COLUMN_STATS) { + rowCount = Statistic.NO_COLUMN_STATS; break; } else { rowCount += statRowCount; @@ -50,7 +49,7 @@ public enum TableStatisticsKind implements CollectableTableStatisticsKind { @Override public Long getValue(BaseMetadata metadata) { Long rowCount = (Long) metadata.getStatistic(this); - return rowCount != null ? rowCount : GroupScan.NO_COLUMN_STATS; + return rowCount != null ? rowCount : Statistic.NO_COLUMN_STATS; } @Override diff --git a/metastore/pom.xml b/metastore/pom.xml new file mode 100644 index 0000000..2558e11 --- /dev/null +++ b/metastore/pom.xml @@ -0,0 +1,75 @@ +<?xml version="1.0"?> +<!-- + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +--> +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + + <parent> + <groupId>org.apache.drill</groupId> + <artifactId>drill-root</artifactId> + <version>1.17.0-SNAPSHOT</version> + </parent> + + <groupId>org.apache.drill.metastore</groupId> + <artifactId>metastore-parent</artifactId> + <packaging>pom</packaging> + <name>metastore/Parent Pom</name> + + <dependencies> + <dependency> + <groupId>org.apache.drill</groupId> + <artifactId>drill-common</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>org.apache.drill</groupId> + <artifactId>drill-logical</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>org.apache.drill.exec</groupId> + <artifactId>vector</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-common</artifactId> + <exclusions> + <exclusion> + <groupId>commons-codec</groupId> + <artifactId>commons-codec</artifactId> + </exclusion> + <exclusion> + <groupId>io.netty</groupId> + <artifactId>netty</artifactId> + </exclusion> + <exclusion> + <groupId>io.netty</groupId> + <artifactId>netty-all</artifactId> + </exclusion> + </exclusions> + </dependency> + </dependencies> + + <modules> + <module>metastore-api</module> + <module>file-metadata</module> + </modules> +</project> diff --git a/pom.xml b/pom.xml index ac9af89..2080964 100644 --- a/pom.xml +++ b/pom.xml @@ -3586,5 +3586,6 @@ <module>exec</module> <module>drill-yarn</module> <module>distribution</module> + <module>metastore</module> </modules> </project>
