[
https://issues.apache.org/jira/browse/DRILL-7271?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16871129#comment-16871129
]
ASF GitHub Bot commented on DRILL-7271:
---------------------------------------
arina-ielchiieva commented on pull request #1810: DRILL-7271: Refactor Metadata
interfaces and classes to contain all needed information for the File based
Metastore
URL: https://github.com/apache/drill/pull/1810#discussion_r296622444
##########
File path:
exec/java-exec/src/main/java/org/apache/drill/exec/expr/StatisticsProvider.java
##########
@@ -218,89 +201,85 @@ public ColumnStatistics
visitFunctionHolderExpression(FunctionHolderExpression h
ValueHolder minFuncHolder =
InterpreterEvaluator.evaluateFunction(interpreter, args1, holderExpr.getName());
ValueHolder maxFuncHolder =
InterpreterEvaluator.evaluateFunction(interpreter, args2, holderExpr.getName());
- MinMaxStatistics statistics;
switch (destType) {
case INT:
- statistics = new MinMaxStatistics<>(((IntHolder)
minFuncHolder).value, ((IntHolder) maxFuncHolder).value, Integer::compareTo);
- break;
+ return StatisticsProvider.getColumnStatistics(
+ ((IntHolder) minFuncHolder).value,
+ ((IntHolder) maxFuncHolder).value,
+ ColumnStatisticsKind.NULLS_COUNT.getFrom(input),
+ destType);
case BIGINT:
- statistics = new MinMaxStatistics<>(((BigIntHolder)
minFuncHolder).value, ((BigIntHolder) maxFuncHolder).value, Long::compareTo);
- break;
+ return StatisticsProvider.getColumnStatistics(
+ ((BigIntHolder) minFuncHolder).value,
+ ((BigIntHolder) maxFuncHolder).value,
+ ColumnStatisticsKind.NULLS_COUNT.getFrom(input),
+ destType);
case FLOAT4:
- statistics = new MinMaxStatistics<>(((Float4Holder)
minFuncHolder).value, ((Float4Holder) maxFuncHolder).value, Float::compareTo);
- break;
+ return StatisticsProvider.getColumnStatistics(
+ ((Float4Holder) minFuncHolder).value,
+ ((Float4Holder) maxFuncHolder).value,
+ ColumnStatisticsKind.NULLS_COUNT.getFrom(input),
+ destType);
case FLOAT8:
- statistics = new MinMaxStatistics<>(((Float8Holder)
minFuncHolder).value, ((Float8Holder) maxFuncHolder).value, Double::compareTo);
- break;
+ return StatisticsProvider.getColumnStatistics(
+ ((Float8Holder) minFuncHolder).value,
+ ((Float8Holder) maxFuncHolder).value,
+ ColumnStatisticsKind.NULLS_COUNT.getFrom(input),
+ destType);
case TIMESTAMP:
- statistics = new MinMaxStatistics<>(((TimeStampHolder)
minFuncHolder).value, ((TimeStampHolder) maxFuncHolder).value, Long::compareTo);
- break;
+ return StatisticsProvider.getColumnStatistics(
+ ((TimeStampHolder) minFuncHolder).value,
+ ((TimeStampHolder) maxFuncHolder).value,
+ ColumnStatisticsKind.NULLS_COUNT.getFrom(input),
+ destType);
default:
return null;
}
- statistics.setNullsCount((long)
input.getStatistic(ColumnStatisticsKind.NULLS_COUNT));
- return statistics;
} catch (Exception e) {
- throw new DrillRuntimeException("Error in evaluating function of " +
holderExpr.getName() );
+ throw new DrillRuntimeException("Error in evaluating function of " +
holderExpr.getName());
}
}
- public static class MinMaxStatistics<V> implements ColumnStatistics<V> {
- private final V minVal;
- private final V maxVal;
- private final Comparator<V> valueComparator;
- private long nullsCount;
-
- public MinMaxStatistics(V minVal, V maxVal, Comparator<V> valueComparator)
{
- this.minVal = minVal;
- this.maxVal = maxVal;
- this.valueComparator = valueComparator;
- }
-
- @Override
- public Object getStatistic(StatisticsKind statisticsKind) {
- switch (statisticsKind.getName()) {
- case ExactStatisticsConstants.MIN_VALUE:
- return minVal;
- case ExactStatisticsConstants.MAX_VALUE:
- return maxVal;
- case ExactStatisticsConstants.NULLS_COUNT:
- return nullsCount;
- default:
- return null;
- }
- }
-
- @Override
- public boolean containsStatistic(StatisticsKind statisticsKind) {
- switch (statisticsKind.getName()) {
- case ExactStatisticsConstants.MIN_VALUE:
- case ExactStatisticsConstants.MAX_VALUE:
- case ExactStatisticsConstants.NULLS_COUNT:
- return true;
- default:
- return false;
- }
- }
-
- @Override
- public boolean containsExactStatistics(StatisticsKind statisticsKind) {
- return true;
- }
-
- @Override
- public Comparator<V> getValueComparator() {
- return valueComparator;
- }
+ /**
+ * Returns {@link ColumnStatistics} instance with set min, max values and
nulls count statistics specified in the arguments.
+ *
+ * @param minVal min value
+ * @param maxVal max value
+ * @param nullsCount nulls count
+ * @param type type of the column
+ * @param <V> type of min and max values
+ * @return {@link ColumnStatistics} instance with set min, max values and
nulls count statistics
+ */
+ public static <V> ColumnStatistics<V> getColumnStatistics(V minVal, V
maxVal, long nullsCount, TypeProtos.MinorType type) {
+ return new ColumnStatistics<>(
+ Arrays.asList(new StatisticsHolder<>(minVal,
ColumnStatisticsKind.MIN_VALUE),
+ new StatisticsHolder<>(maxVal, ColumnStatisticsKind.MAX_VALUE),
+ new StatisticsHolder<>(nullsCount,
ColumnStatisticsKind.NULLS_COUNT)),
+ type);
+ }
- @Override
- public ColumnStatistics<V> cloneWithStats(ColumnStatistics statistics) {
- throw new UnsupportedOperationException("MinMaxStatistics does not
support cloneWithStats");
- }
+ /**
+ * Returns {@link ColumnStatistics} instance with min and max values set to
{@code minMaxValue} and 0 nulls count statistics.
Review comment:
1. Re-phase: `and nulls count sets to 0`
2. Please explain why there cannot be any nulls.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
> Refactor Metadata interfaces and classes to contain all needed information
> for the File based Metastore
> -------------------------------------------------------------------------------------------------------
>
> Key: DRILL-7271
> URL: https://issues.apache.org/jira/browse/DRILL-7271
> Project: Apache Drill
> Issue Type: Sub-task
> Reporter: Arina Ielchiieva
> Assignee: Volodymyr Vysotskyi
> Priority: Major
> Fix For: 1.17.0
>
>
> 1. Merge info from metadataStatistics + statisticsKinds into one holder:
> Map<String, StatisticsHolder>.
> 2. Rename hasStatistics to hasDescriptiveStatistics
> 3. Remove drill-file-metastore-plugin
> 4. Move
> org.apache.drill.exec.physical.base.AbstractGroupScanWithMetadata.MetadataLevel
> to metadata module, rename to MetadataType and add new value: SEGMENT.
> 5. Add JSON ser/de for ColumnStatistics, StatisticsHolder.
> 6. Add new info classes:
> {noformat}
> class TableInfo {
> String storagePlugin;
> String workspace;
> String name;
> String type;
> String owner;
> }
> class MetadataInfo {
> public static final String GENERAL_INFO_KEY = "GENERAL_INFO";
> public static final String DEFAULT_SEGMENT_KEY = "DEFAULT_SEGMENT";
> MetadataType type (enum);
> String key;
> String identifier;
> }
> {noformat}
> 7. Modify existing metadata classes:
> org.apache.drill.metastore.FileTableMetadata
> {noformat}
> missing fields
> ------------------
> storagePlugin, workspace, tableType -> will be covered by TableInfo class
> metadataType, metadataKey -> will be covered by MetadataInfo class
> interestingColumns
> fields to modify
> ----------------
> private final Map<String, Object> tableStatistics;
> private final Map<String, StatisticsKind> statisticsKinds;
> private final Set<String> partitionKeys; -> Map<String, String>
> {noformat}
> org.apache.drill.metastore.PartitionMetadata
> {noformat}
> missing fields
> ------------------
> storagePlugin, workspace -> will be covered by TableInfo class
> metadataType, metadataKey, metadataIdentifier -> will be covered by
> MetadataInfo class
> partitionValues (List<String>)
> location (String) (for directory level metadata) - directory location
> fields to modify
> ----------------
> private final Map<String, Object> tableStatistics;
> private final Map<String, StatisticsKind> statisticsKinds;
> private final Set<Path> location; -> locations
> {noformat}
> org.apache.drill.metastore.FileMetadata
> {noformat}
> missing fields
> ------------------
> storagePlugin, workspace -> will be covered by TableInfo class
> metadataType, metadataKey, metadataIdentifier -> will be covered by
> MetadataInfo class
> path - path to file
> fields to modify
> ----------------
> private final Map<String, Object> tableStatistics;
> private final Map<String, StatisticsKind> statisticsKinds;
> private final Path location; - should contain directory to which file belongs
> {noformat}
> org.apache.drill.metastore.RowGroupMetadata
> {noformat}
> missing fields
> ------------------
> storagePlugin, workspace -> will be covered by TableInfo class
> metadataType, metadataKey, metadataIdentifier -> will be covered by
> MetadataInfo class
> path - path to file
> fields to modify
> ----------------
> private final Map<String, Object> tableStatistics;
> private final Map<String, StatisticsKind> statisticsKinds;
> private final Path location; - should contain directory to which file belongs
> {noformat}
> 8. Remove org.apache.drill.exec package from metastore module.
> 9. Rename ColumnStatisticsImpl class.
> 10. Separate existing classes in org.apache.drill.metastore package into
> sub-packages.
> 11. Rename FileTableMetadata -> BaseTableMetadata
> 12. TableMetadataProvider.getNonInterestingColumnsMeta() ->
> getNonInterestingColumnsMetadata
> 13. Introduce segment-level metadata class:
> {noformat}
> class SegmentMetadata {
> TableInfo tableInfo;
> MetadataInfo metadataInfo;
> SchemaPath column;
> TupleMetadata schema;
> String location;
> Map<SchemaPath, ColumnStatistics> columnsStatistics;
> Map<String, StatisticsHolder> statistics;
> List<String> partitionValues;
> List<String> locations;
> long lastModifiedTime;
> }
> {noformat}
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
