clintropolis commented on code in PR #20452: URL: https://github.com/apache/druid/pull/20452#discussion_r4161268186
########## server/src/main/java/org/apache/druid/catalog/model/RollupTableBaseTableMetadata.java: ########## @@ -0,0 +1,232 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.catalog.model; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonTypeName; +import org.apache.druid.data.input.impl.DimensionSchema; +import org.apache.druid.data.input.impl.RollupTableProjectionSpec; +import org.apache.druid.error.InvalidInput; +import org.apache.druid.query.aggregation.AggregatorFactory; +import org.apache.druid.segment.VirtualColumns; +import org.apache.druid.segment.column.ColumnType; +import org.apache.druid.utils.CollectionUtils; + +import javax.annotation.Nullable; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; + +/** + * Catalog layout metadata for {@link RollupTableProjectionSpec} base tables. The catalog column list declares the + * output schema of the rollup: the columns named by an entry of {@link #aggregators} are the metric columns, every + * other declared column is a grouping column, and, mirroring the physical layout, the grouping columns must be + * declared before the metric columns. {@link #createSpec(List)} combines the declared columns with this metadata into + * the physical spec used to generate segments; as with the other layouts, the optional {@link #virtualColumns} carry + * only the query granularity, and the optional {@link #columnSchemas} customize the {@link DimensionSchema} used for a + * declared grouping column (a metric column's physical representation is fixed by its aggregator). + */ +@JsonTypeName(RollupTableBaseTableMetadata.TYPE_NAME) +public class RollupTableBaseTableMetadata implements DatasourceBaseTableMetadata +{ + public static final String TYPE_NAME = RollupTableProjectionSpec.TYPE_NAME; + + private final VirtualColumns virtualColumns; + private final AggregatorFactory[] aggregators; + private final List<DimensionSchema> columnSchemas; + + @JsonCreator + public RollupTableBaseTableMetadata( + @JsonProperty("virtualColumns") @Nullable VirtualColumns virtualColumns, + @JsonProperty("aggregators") @Nullable AggregatorFactory[] aggregators, + @JsonProperty("columnSchemas") @Nullable List<DimensionSchema> columnSchemas + ) + { + this.virtualColumns = virtualColumns == null ? VirtualColumns.EMPTY : virtualColumns; + this.aggregators = aggregators == null ? new AggregatorFactory[0] : aggregators; + this.columnSchemas = columnSchemas == null ? Collections.emptyList() : columnSchemas; + } + + @Override + @JsonProperty("type") + public String getType() + { + return TYPE_NAME; + } + + @Override + @JsonProperty("virtualColumns") + @JsonInclude(JsonInclude.Include.NON_DEFAULT) + public VirtualColumns getVirtualColumns() + { + return virtualColumns; + } + + /** + * The aggregators computed over the rows collapsed into each grouping tuple. Each names the declared metric column + * it fills. + */ + @JsonProperty("aggregators") + @JsonInclude(JsonInclude.Include.NON_EMPTY) + public AggregatorFactory[] getAggregators() + { + return aggregators; + } + + /** + * Per-column customizations of the {@link DimensionSchema} used during segment creation, keyed by + * {@link DimensionSchema#getName()}; empty when every declared grouping column uses the schema derived from its + * declared type. These do not define columns: every entry must customize a declared grouping column. + */ + @JsonProperty("columnSchemas") + @JsonInclude(JsonInclude.Include.NON_EMPTY) + public List<DimensionSchema> getColumnSchemas() + { + return columnSchemas; + } + + /** + * Creates the physical spec from the declared catalog columns: a declared column named by an aggregator is a metric + * column, whose declared type must match what its aggregator produces; every other declared column is a grouping + * column, derived like the other layouts derive theirs. Declared column order is the physical segment order, so the + * grouping columns must be declared before the metric columns; the metric columns' declared order becomes the + * aggregator order of the spec. + */ + @Override + public RollupTableProjectionSpec createSpec(List<ColumnSpec> columns) + { + if (CollectionUtils.isNullOrEmpty(columns)) { + throw InvalidInput.exception( + "Cannot define a [%s] base table without declared columns; the catalog column list defines the table schema", + TYPE_NAME + ); + } + final Map<String, AggregatorFactory> aggregatorsByName = new HashMap<>(); + for (AggregatorFactory aggregator : aggregators) { + if (aggregator == null) { + throw InvalidInput.exception("aggregators must not contain null entries"); + } + if (aggregatorsByName.put(aggregator.getName(), aggregator) != null) { + throw InvalidInput.exception("aggregators contains duplicate entries for column [%s]", aggregator.getName()); + } + } Review Comment: this is still called as part of `DatasourceDefn.validate` so these checks are done whenever making any catalog write, it is just that base table spec defined in the catalog need the table level column definition list to be able to validate since they are allowed to be partial declarations (column schemas are optional and only need to provide explicit dimension schemas when there is some customization that needs done during indexing) and this also ensures that the table level column lists and projection column lists agree. Doing it all here ensures that if a bad spec does make it into the catalog somehow it is still fixable (because the validation belongs to write). -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
