[
https://issues.apache.org/jira/browse/DRILL-7271?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16871454#comment-16871454
]
ASF GitHub Bot commented on DRILL-7271:
---------------------------------------
vvysotskyi commented on pull request #1810: DRILL-7271: Refactor Metadata
interfaces and classes to contain all needed information for the File based
Metastore
URL: https://github.com/apache/drill/pull/1810#discussion_r296752797
##########
File path:
metastore/metastore-api/src/main/java/org/apache/drill/metastore/metadata/BaseMetadata.java
##########
@@ -0,0 +1,148 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.metastore.metadata;
+
+import org.apache.drill.common.expression.SchemaPath;
+import org.apache.drill.exec.record.metadata.ColumnMetadata;
+import org.apache.drill.exec.record.metadata.TupleMetadata;
+import org.apache.drill.metastore.SchemaPathUtils;
+import org.apache.drill.metastore.statistics.ColumnStatistics;
+import org.apache.drill.metastore.statistics.StatisticsHolder;
+import org.apache.drill.metastore.statistics.StatisticsKind;
+
+import java.util.Collection;
+import java.util.Map;
+import java.util.Objects;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+/**
+ * Common provider of tuple schema, column metadata, and statistics for table,
partition, file or row group.
+ */
+public abstract class BaseMetadata implements Metadata {
+ protected final TableInfo tableInfo;
+ protected final MetadataInfo metadataInfo;
+ protected final TupleMetadata schema;
+ protected final Map<SchemaPath, ColumnStatistics> columnsStatistics;
+ protected final Map<String, StatisticsHolder> statistics;
+
+ protected <T extends BaseMetadataBuilder<T>>
BaseMetadata(BaseMetadataBuilder<T> builder) {
+ this.tableInfo = builder.tableInfo;
+ this.metadataInfo = builder.metadataInfo;
+ this.schema = builder.schema;
+ this.columnsStatistics = builder.columnsStatistics;
+ this.statistics = builder.statistics.stream()
+ .collect(Collectors.toMap(
+ statistic -> statistic.getStatisticsKind().getName(),
+ Function.identity(),
+ (a, b) -> a.getStatisticsKind().isExact() ? a : b));
+ }
+
+ @Override
+ public Map<SchemaPath, ColumnStatistics> getColumnsStatistics() {
+ return columnsStatistics;
+ }
+
+ @Override
+ public ColumnStatistics getColumnStatistics(SchemaPath columnName) {
+ return columnsStatistics.get(columnName);
+ }
+
+ @Override
+ public TupleMetadata getSchema() {
+ return schema;
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public <V> V getStatistic(StatisticsKind<V> statisticsKind) {
+ StatisticsHolder<V> statisticsHolder =
statistics.get(statisticsKind.getName());
+ return statisticsHolder != null ? statisticsHolder.getStatisticsValue() :
null;
+ }
+
+ @Override
+ public boolean containsExactStatistics(StatisticsKind statisticsKind) {
+ StatisticsHolder statisticsHolder =
statistics.get(statisticsKind.getName());
+ return statisticsHolder != null &&
statisticsHolder.getStatisticsKind().isExact();
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public <V> V getStatisticsForColumn(SchemaPath columnName, StatisticsKind<V>
statisticsKind) {
+ return (V) columnsStatistics.get(columnName).get(statisticsKind);
+ }
+
+ @Override
+ public ColumnMetadata getColumn(SchemaPath name) {
+ return SchemaPathUtils.getColumnMetadata(name, schema);
+ }
+
+ @Override
+ public TableInfo getTableInfo() {
+ return tableInfo;
+ }
+
+ @Override
+ public MetadataInfo getMetadataInfo() {
+ return metadataInfo;
+ }
+
+ public static abstract class BaseMetadataBuilder<T extends
BaseMetadataBuilder<T>> {
+ protected TableInfo tableInfo;
+ protected MetadataInfo metadataInfo;
+ protected TupleMetadata schema;
+ protected Map<SchemaPath, ColumnStatistics> columnsStatistics;
+ protected Collection<StatisticsHolder> statistics;
+
+ public T withTableInfo(TableInfo tableInfo) {
Review comment:
Thanks, removed.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
> Refactor Metadata interfaces and classes to contain all needed information
> for the File based Metastore
> -------------------------------------------------------------------------------------------------------
>
> Key: DRILL-7271
> URL: https://issues.apache.org/jira/browse/DRILL-7271
> Project: Apache Drill
> Issue Type: Sub-task
> Reporter: Arina Ielchiieva
> Assignee: Volodymyr Vysotskyi
> Priority: Major
> Fix For: 1.17.0
>
>
> 1. Merge info from metadataStatistics + statisticsKinds into one holder:
> Map<String, StatisticsHolder>.
> 2. Rename hasStatistics to hasDescriptiveStatistics
> 3. Remove drill-file-metastore-plugin
> 4. Move
> org.apache.drill.exec.physical.base.AbstractGroupScanWithMetadata.MetadataLevel
> to metadata module, rename to MetadataType and add new value: SEGMENT.
> 5. Add JSON ser/de for ColumnStatistics, StatisticsHolder.
> 6. Add new info classes:
> {noformat}
> class TableInfo {
> String storagePlugin;
> String workspace;
> String name;
> String type;
> String owner;
> }
> class MetadataInfo {
> public static final String GENERAL_INFO_KEY = "GENERAL_INFO";
> public static final String DEFAULT_SEGMENT_KEY = "DEFAULT_SEGMENT";
> MetadataType type (enum);
> String key;
> String identifier;
> }
> {noformat}
> 7. Modify existing metadata classes:
> org.apache.drill.metastore.FileTableMetadata
> {noformat}
> missing fields
> ------------------
> storagePlugin, workspace, tableType -> will be covered by TableInfo class
> metadataType, metadataKey -> will be covered by MetadataInfo class
> interestingColumns
> fields to modify
> ----------------
> private final Map<String, Object> tableStatistics;
> private final Map<String, StatisticsKind> statisticsKinds;
> private final Set<String> partitionKeys; -> Map<String, String>
> {noformat}
> org.apache.drill.metastore.PartitionMetadata
> {noformat}
> missing fields
> ------------------
> storagePlugin, workspace -> will be covered by TableInfo class
> metadataType, metadataKey, metadataIdentifier -> will be covered by
> MetadataInfo class
> partitionValues (List<String>)
> location (String) (for directory level metadata) - directory location
> fields to modify
> ----------------
> private final Map<String, Object> tableStatistics;
> private final Map<String, StatisticsKind> statisticsKinds;
> private final Set<Path> location; -> locations
> {noformat}
> org.apache.drill.metastore.FileMetadata
> {noformat}
> missing fields
> ------------------
> storagePlugin, workspace -> will be covered by TableInfo class
> metadataType, metadataKey, metadataIdentifier -> will be covered by
> MetadataInfo class
> path - path to file
> fields to modify
> ----------------
> private final Map<String, Object> tableStatistics;
> private final Map<String, StatisticsKind> statisticsKinds;
> private final Path location; - should contain directory to which file belongs
> {noformat}
> org.apache.drill.metastore.RowGroupMetadata
> {noformat}
> missing fields
> ------------------
> storagePlugin, workspace -> will be covered by TableInfo class
> metadataType, metadataKey, metadataIdentifier -> will be covered by
> MetadataInfo class
> path - path to file
> fields to modify
> ----------------
> private final Map<String, Object> tableStatistics;
> private final Map<String, StatisticsKind> statisticsKinds;
> private final Path location; - should contain directory to which file belongs
> {noformat}
> 8. Remove org.apache.drill.exec package from metastore module.
> 9. Rename ColumnStatisticsImpl class.
> 10. Separate existing classes in org.apache.drill.metastore package into
> sub-packages.
> 11. Rename FileTableMetadata -> BaseTableMetadata
> 12. TableMetadataProvider.getNonInterestingColumnsMeta() ->
> getNonInterestingColumnsMetadata
> 13. Introduce segment-level metadata class:
> {noformat}
> class SegmentMetadata {
> TableInfo tableInfo;
> MetadataInfo metadataInfo;
> SchemaPath column;
> TupleMetadata schema;
> String location;
> Map<SchemaPath, ColumnStatistics> columnsStatistics;
> Map<String, StatisticsHolder> statistics;
> List<String> partitionValues;
> List<String> locations;
> long lastModifiedTime;
> }
> {noformat}
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
