This is an automated email from the ASF dual-hosted git repository.
clintropolis pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/master by this push:
new 443688fea12 feat: add plain table base table spec for catalog wiring
(#20377)
443688fea12 is described below
commit 443688fea12b1fee730f38f3aef87462573b475e
Author: Clint Wylie <[email protected]>
AuthorDate: Tue Sep 29 15:30:36 2026 -0700
feat: add plain table base table spec for catalog wiring (#20377)
---
.../embedded/catalog/CatalogDdlAndIngestTest.java | 96 +++++
.../testing/embedded/indexing/IndexTaskTest.java | 56 +++
.../data/input/impl/AggregateProjectionSpec.java | 8 +-
.../data/input/impl/BaseTableProjectionSpec.java | 50 ++-
...lusteredValueGroupsBaseTableProjectionSpec.java | 9 +-
.../druid/data/input/impl/TableProjectionSpec.java | 360 +++++++++++++++++
.../util/common/granularity/Granularities.java | 31 +-
.../org/apache/druid/segment/VirtualColumn.java | 20 +
.../segment/virtual/ExpressionVirtualColumn.java | 12 +
.../segment/virtual/NestedFieldVirtualColumn.java | 20 +
.../segment/virtual/NestedMergeVirtualColumn.java | 6 +
.../segment/virtual/NestedObjectVirtualColumn.java | 6 +
.../SpecializedExpressionVirtualColumn.java | 13 +
.../input/impl/AggregateProjectionSpecTest.java | 21 +
...eredValueGroupsBaseTableProjectionSpecTest.java | 105 +++++
.../data/input/impl/TableProjectionSpecTest.java | 428 +++++++++++++++++++++
.../virtual/ExpressionVirtualColumnTest.java | 12 +
.../virtual/NestedFieldVirtualColumnTest.java | 14 +
.../virtual/NestedMergeVirtualColumnTest.java | 17 +
.../virtual/NestedObjectVirtualColumnTest.java | 16 +
.../druid/catalog/model/BaseTableColumns.java | 166 ++++++++
.../ClusteredValueGroupsBaseTableMetadata.java | 137 +------
.../catalog/model/DatasourceBaseTableMetadata.java | 1 +
.../catalog/model/TableBaseTableMetadata.java | 164 ++++++++
.../apache/druid/segment/indexing/DataSchema.java | 5 +-
.../catalog/model/TableBaseTableMetadataTest.java | 190 +++++++++
.../druid/segment/indexing/DataSchemaTest.java | 42 ++
.../sql/calcite/planner/CatalogDdlHandler.java | 4 +-
.../calcite/planner/ProjectionSpecTranslator.java | 180 +++++++--
.../druid/sql/calcite/CalciteCatalogDdlTest.java | 180 +++++++++
.../sql/calcite/parser/DruidSqlDdlParserTest.java | 5 +
31 files changed, 2197 insertions(+), 177 deletions(-)
diff --git
a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/catalog/CatalogDdlAndIngestTest.java
b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/catalog/CatalogDdlAndIngestTest.java
index 9d021e8e7d5..b280a7a15a0 100644
---
a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/catalog/CatalogDdlAndIngestTest.java
+++
b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/catalog/CatalogDdlAndIngestTest.java
@@ -407,6 +407,102 @@ public class CatalogDdlAndIngestTest extends
CatalogTestBase
);
}
+ /**
+ * A computed column composed of specialized expressions is lifted with its
dependency closure (the inner JSON_MERGE
+ * becomes an intermediary virtual column the outer nested-field virtual
column reads by synthetic name), and the
+ * clustered write path evaluates the chain at ingest to materialize the
declared column.
+ */
+ @Test
+ public void testCreateClusteredBaseTableWithComposedComputedColumn()
+ {
+ final String tableName = dataSource;
+
+ cluster.callApi().runSql(
+ "CREATE TABLE \"%s\" (\n"
+ + " tenant VARCHAR,\n"
+ + " __time TIMESTAMP,\n"
+ + " payload TYPE('COMPLEX<json>'),\n"
+ + " v BIGINT,\n"
+ + " PROJECTION __base AS (\n"
+ + " SELECT tenant, __time, payload, JSON_VALUE(JSON_MERGE(payload,
payload), '$.v' RETURNING BIGINT) AS v\n"
+ + " CLUSTERED BY tenant\n"
+ + " )\n"
+ + ")\n"
+ + "PARTITIONED BY DAY",
+ tableName
+ );
+
+ // 'v' is computed by the base table, so the INSERT supplies only its
input column.
+ ingest(
+ "INSERT INTO \"%s\"\n"
+ + "SELECT TIME_PARSE(t) AS __time, a AS tenant, p AS payload\n"
+ + "FROM TABLE(\n"
+ + " EXTERN(\n"
+ + " '{\"type\":\"inline\",\"data\":\""
+ +
"{\\\"t\\\":\\\"2022-12-26T12:34:56\\\",\\\"a\\\":\\\"bbb\\\",\\\"p\\\":{\\\"v\\\":7}}"
+ +
"\\n{\\\"t\\\":\\\"2022-12-26T12:34:56\\\",\\\"a\\\":\\\"aaa\\\",\\\"p\\\":{\\\"v\\\":12}}\"}',\n"
+ + " '{\"type\":\"json\"}'\n"
+ + " )\n"
+ + ") EXTEND (t VARCHAR, a VARCHAR, p TYPE('COMPLEX<json>'))\n",
+ tableName
+ );
+
+ // Rows come back in clustering order, with 'v' materialized at ingest by
the composed virtual column chain.
+ cluster.callApi().verifySqlQuery(
+ "SELECT tenant, v FROM %s",
+ tableName,
+ "aaa,12\n"
+ + "bbb,7"
+ );
+ }
+
+ /**
+ * A {@code __base} projection without {@code CLUSTERED BY} declares the
plain-table layout: declared column order is
+ * the segment storage and sort order, and {@code TIME_FLOOR(__time,
<period>)} declares the table's query
+ * granularity. Rows arrive unsorted with unfloored timestamps and come back
sorted by the declared order (item
+ * before {@code __time}) with {@code __time} floored to the hour.
+ */
+ @Test
+ public void testCreatePlainBaseTableThenIngestAndQuery()
+ {
+ final String tableName = dataSource;
+
+ cluster.callApi().runSql(
+ "CREATE TABLE \"%s\" (\n"
+ + " item VARCHAR,\n"
+ + " __time TIMESTAMP,\n"
+ + " val BIGINT,\n"
+ + " PROJECTION __base AS (SELECT item, TIME_FLOOR(__time, 'PT1H') AS
__time, val)\n"
+ + ")\n"
+ + "PARTITIONED BY DAY",
+ tableName
+ );
+
+ final TableMetadata table =
client.readTable(TableId.datasource(tableName));
+
assertNotNull(table.spec().properties().get(DatasourceDefn.BASE_TABLE_PROPERTY));
+
+ ingest(
+ "INSERT INTO \"%s\"\n"
+ + "SELECT TIME_PARSE(a) AS __time, b AS item, c AS val\n"
+ + "FROM TABLE(\n"
+ + " EXTERN(\n"
+ + " '{\"type\":\"inline\",\"data\":\"2022-12-26T05:30:10,cherry,3"
+ +
"\\n2022-12-26T01:45:20,banana,2\\n2022-12-26T09:10:30,apple,1\"}',\n"
+ + "
'{\"type\":\"csv\",\"findColumnsFromHeader\":false,\"columns\":[\"a\",\"b\",\"c\"]}'\n"
+ + " )\n"
+ + ") EXTEND (a VARCHAR, b VARCHAR, c BIGINT)\n",
+ tableName
+ );
+
+ cluster.callApi().verifySqlQuery(
+ "SELECT * FROM %s",
+ tableName,
+ "apple,2022-12-26T09:00:00.000Z,1\n"
+ + "banana,2022-12-26T01:00:00.000Z,2\n"
+ + "cherry,2022-12-26T05:00:00.000Z,3"
+ );
+ }
+
/**
* The same layout without SEALED: a column the query produces but the table
does not declare is stored after the
* declared layout rather than rejected or dropped, and the clustering the
table declared is unchanged.
diff --git
a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexing/IndexTaskTest.java
b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexing/IndexTaskTest.java
index bde5c512a36..2b63fde274a 100644
---
a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexing/IndexTaskTest.java
+++
b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexing/IndexTaskTest.java
@@ -19,9 +19,15 @@
package org.apache.druid.testing.embedded.indexing;
+import org.apache.druid.data.input.impl.LongDimensionSchema;
+import org.apache.druid.data.input.impl.StringDimensionSchema;
+import org.apache.druid.data.input.impl.TableProjectionSpec;
+import org.apache.druid.indexer.granularity.SegmentGranularitySpec;
import org.apache.druid.indexing.common.task.IndexTask;
+import org.apache.druid.indexing.common.task.TaskBuilder;
import org.apache.druid.java.util.common.DateTimes;
import org.apache.druid.java.util.common.StringUtils;
+import org.apache.druid.java.util.common.granularity.Granularities;
import org.apache.druid.java.util.common.guava.Comparators;
import org.apache.druid.testing.embedded.EmbeddedBroker;
import org.apache.druid.testing.embedded.EmbeddedClusterApis;
@@ -106,6 +112,56 @@ public class IndexTaskTest extends EmbeddedClusterTestBase
Assertions.assertEquals("10", cluster.runSql("SELECT COUNT(*) FROM %s",
dataSource));
}
+ /**
+ * Batch ingestion driven by a plain {@link TableProjectionSpec} base table
instead of the legacy top-level schema
+ * fields. The spec lowers into the standard ingest inputs, so declared
column order becomes segment sort order
+ * (item before __time here — not the legacy time-first sort) and the spec's
query-granularity carrier floors
+ * {@code __time}, both observable in the ingested segment.
+ */
+ @Test
+ @Timeout(60)
+ public void test_runIndexTask_withBaseTableSpec()
+ {
+ final String taskId = EmbeddedClusterApis.newTaskId(dataSource);
+ final TableProjectionSpec baseTable = TableProjectionSpec
+ .builder()
+ .columns(
+ new StringDimensionSchema("item"),
+ new LongDimensionSchema("__time"),
+ new LongDimensionSchema("value")
+ )
+ .build()
+ .withQueryGranularity(Granularities.HOUR);
+ final IndexTask task = TaskBuilder
+ .ofTypeIndex()
+ .isoTimestampColumn("time")
+ .csvInputFormatWithColumns("time", "item", "value")
+ .inlineInputSourceWithData(
+ "2025-06-01T05:30:00.000Z,cherry,3"
+ + "\n2025-06-01T01:45:00.000Z,banana,2"
+ + "\n2025-06-01T09:10:00.000Z,apple,1"
+ )
+ .dataSchema(
+ schema -> schema
+ .withBaseTable(baseTable)
+ .withSegmentGranularity(new
SegmentGranularitySpec(Granularities.DAY, null))
+ )
+ .dataSource(dataSource)
+ .withId(taskId);
+
+ cluster.callApi().runTask(task, overlord);
+ cluster.callApi().waitForAllSegmentsToBeAvailable(dataSource, coordinator,
broker);
+
+ // A scan without ORDER BY returns rows in segment order: sorted by item
first (declared order), with __time
+ // floored to the hour by the carrier, even though the rows arrived
unsorted with unfloored timestamps.
+ Assertions.assertEquals(
+ "apple,1,2025-06-01T09:00:00.000Z"
+ + "\nbanana,2,2025-06-01T01:00:00.000Z"
+ + "\ncherry,3,2025-06-01T05:00:00.000Z",
+ cluster.runSql("SELECT item, \"value\", __time FROM %s", dataSource)
+ );
+ }
+
@Test
@Timeout(20)
public void test_run10Tasks_concurrently()
diff --git
a/processing/src/main/java/org/apache/druid/data/input/impl/AggregateProjectionSpec.java
b/processing/src/main/java/org/apache/druid/data/input/impl/AggregateProjectionSpec.java
index a19a5e1a6e6..b60106dca4e 100644
---
a/processing/src/main/java/org/apache/druid/data/input/impl/AggregateProjectionSpec.java
+++
b/processing/src/main/java/org/apache/druid/data/input/impl/AggregateProjectionSpec.java
@@ -30,7 +30,6 @@ import org.apache.druid.error.DruidException;
import org.apache.druid.error.InvalidInput;
import org.apache.druid.java.util.common.granularity.Granularities;
import org.apache.druid.java.util.common.granularity.Granularity;
-import org.apache.druid.java.util.common.granularity.PeriodGranularity;
import org.apache.druid.query.OrderBy;
import org.apache.druid.query.aggregation.AggregatorFactory;
import org.apache.druid.query.filter.DimFilter;
@@ -41,7 +40,6 @@ import org.apache.druid.segment.column.ValueType;
import org.apache.druid.segment.projections.AggregateProjectionSchema;
import org.apache.druid.segment.projections.Projections;
import org.apache.druid.utils.CollectionUtils;
-import org.joda.time.DateTimeZone;
import javax.annotation.Nullable;
import java.util.Arrays;
@@ -248,16 +246,14 @@ public class AggregateProjectionSpec
continue;
}
final VirtualColumn vc =
virtualColumns.getVirtualColumn(groupingColumn.getName());
- final Granularity maybeGranularity =
Granularities.fromVirtualColumn(vc);
+ final Granularity maybeGranularity =
Granularities.fromTimeVirtualColumn(vc);
if (maybeGranularity == null ||
maybeGranularity.equals(Granularities.ALL)) {
// no __time in inputs or not supported, skip
} else if (Granularities.NONE.equals(maybeGranularity)) {
timeColumnName = groupingColumn.getName();
// already found exact __time grouping, skip assigning, granularity
= Granularities.NONE;
break;
- } else if (maybeGranularity.getClass().equals(PeriodGranularity.class)
- && maybeGranularity.getTimeZone().equals(DateTimeZone.UTC)
- && ((PeriodGranularity) maybeGranularity).getOrigin() == null
+ } else if (Granularities.isStandardUtcPeriod(maybeGranularity)
&& (granularity == null ||
maybeGranularity.isFinerThan(granularity))) {
// found a finer period granularity than the existing granularity,
or it's the first one
timeColumnName = groupingColumn.getName();
diff --git
a/processing/src/main/java/org/apache/druid/data/input/impl/BaseTableProjectionSpec.java
b/processing/src/main/java/org/apache/druid/data/input/impl/BaseTableProjectionSpec.java
index 292a26fe68c..e9d42175e4c 100644
---
a/processing/src/main/java/org/apache/druid/data/input/impl/BaseTableProjectionSpec.java
+++
b/processing/src/main/java/org/apache/druid/data/input/impl/BaseTableProjectionSpec.java
@@ -21,15 +21,18 @@ package org.apache.druid.data.input.impl;
import com.fasterxml.jackson.annotation.JsonSubTypes;
import com.fasterxml.jackson.annotation.JsonTypeInfo;
+import org.apache.druid.error.InvalidInput;
import org.apache.druid.java.util.common.granularity.Granularities;
import org.apache.druid.java.util.common.granularity.Granularity;
import org.apache.druid.query.OrderBy;
import org.apache.druid.query.aggregation.AggregatorFactory;
import org.apache.druid.segment.VirtualColumn;
import org.apache.druid.segment.VirtualColumns;
+import org.apache.druid.segment.column.ColumnHolder;
import org.apache.druid.segment.projections.BaseTableProjectionSchema;
import javax.annotation.Nullable;
+import java.util.Collections;
import java.util.List;
/**
@@ -47,6 +50,7 @@ import java.util.List;
*/
@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type")
@JsonSubTypes({
+ @JsonSubTypes.Type(name = TableProjectionSpec.TYPE_NAME, value =
TableProjectionSpec.class),
@JsonSubTypes.Type(
name = ClusteredValueGroupsBaseTableProjectionSpec.TYPE_NAME,
value = ClusteredValueGroupsBaseTableProjectionSpec.class
@@ -72,10 +76,7 @@ public interface BaseTableProjectionSpec
{
final VirtualColumn granularityVirtualColumn =
getVirtualColumns().getVirtualColumn(Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME);
- if (granularityVirtualColumn == null) {
- return Granularities.NONE;
- }
- final Granularity granularity =
Granularities.fromVirtualColumn(granularityVirtualColumn);
+ final Granularity granularity =
Granularities.fromTimeVirtualColumn(granularityVirtualColumn);
return granularity == null ? Granularities.NONE : granularity;
}
@@ -104,4 +105,45 @@ public interface BaseTableProjectionSpec
{
return equals(other);
}
+
+ /**
+ * Validates that {@code queryGranularity} can be used by a base-table spec:
only period granularities in the UTC
+ * time zone without an origin are currently allowed.
+ */
+ static void validateQueryGranularity(Granularity queryGranularity, String
typeName)
+ {
+ if (!Granularities.isStandardUtcPeriod(queryGranularity)) {
+ throw InvalidInput.exception(
+ "Query granularity[%s] is not supported for [%s] base tables; only
period granularities in the UTC time"
+ + " zone without an origin are supported",
+ queryGranularity,
+ typeName
+ );
+ }
+ }
+
+ /**
+ * Validates a {@link Granularities#GRANULARITY_VIRTUAL_COLUMN_NAME}
supplied directly to a spec (catalog JSON, or
+ * a translated DDL body): it must decode to a granularity {@link
#validateQueryGranularity} accepts, so a
+ * spec cannot be constructed claiming a granularity that reads back as
something else.
+ */
+ static void validateGranularity(VirtualColumn granularityColumn, String
typeName)
+ {
+ if
(!Collections.singletonList(ColumnHolder.TIME_COLUMN_NAME).equals(granularityColumn.requiredColumns()))
{
+ throw InvalidInput.exception(
+ "virtual column [%s] must be computed from [%s] alone, but reads %s",
+ Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME,
+ ColumnHolder.TIME_COLUMN_NAME,
+ granularityColumn.requiredColumns()
+ );
+ }
+ final Granularity granularity =
Granularities.fromTimeVirtualColumn(granularityColumn);
+ if (granularity == null || Granularities.NONE.equals(granularity) ||
Granularities.ALL.equals(granularity)) {
+ throw InvalidInput.exception(
+ "virtual column [%s] does not encode a query granularity",
+ Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME
+ );
+ }
+ validateQueryGranularity(granularity, typeName);
+ }
}
diff --git
a/processing/src/main/java/org/apache/druid/data/input/impl/ClusteredValueGroupsBaseTableProjectionSpec.java
b/processing/src/main/java/org/apache/druid/data/input/impl/ClusteredValueGroupsBaseTableProjectionSpec.java
index 4a2855ba9b3..9ed41ee9a1c 100644
---
a/processing/src/main/java/org/apache/druid/data/input/impl/ClusteredValueGroupsBaseTableProjectionSpec.java
+++
b/processing/src/main/java/org/apache/druid/data/input/impl/ClusteredValueGroupsBaseTableProjectionSpec.java
@@ -184,7 +184,8 @@ public final class
ClusteredValueGroupsBaseTableProjectionSpec implements BaseTa
* {@code null} or {@code NONE} granularity is a no-op (no flooring), so
this returns {@code this} unchanged.
* <p>
* {@code ALL} is rejected: it would floor {@code __time} to a single
constant (the interval start) for the whole
- * segment, which clustered base tables do not yet support.
+ * segment, which clustered base tables do not yet support. Everything other
than a period granularity in the UTC
+ * time zone is rejected too, see {@link
BaseTableProjectionSpec#validateQueryGranularity}.
* <p>
* Idempotent: if the spec already declares a query-granularity virtual
column, that one is authoritative and this is
* a no-op. (The compaction path attaches the virtual column up front; the
MSQ generation path then calls this again
@@ -203,6 +204,7 @@ public final class
ClusteredValueGroupsBaseTableProjectionSpec implements BaseTa
||
virtualColumns.getVirtualColumn(Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME)
!= null) {
return this;
}
+ BaseTableProjectionSpec.validateQueryGranularity(queryGranularity,
TYPE_NAME);
final VirtualColumn granularityVirtualColumn =
Granularities.toVirtualColumn(queryGranularity,
Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME);
final List<VirtualColumn> merged = new
ArrayList<>(Arrays.asList(virtualColumns.getVirtualColumns()));
@@ -378,6 +380,11 @@ public final class
ClusteredValueGroupsBaseTableProjectionSpec implements BaseTa
if (all.length == 0) {
return;
}
+ final VirtualColumn granularity =
+
virtualColumns.getVirtualColumn(Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME);
+ if (granularity != null) {
+ BaseTableProjectionSpec.validateGranularity(granularity, TYPE_NAME);
+ }
// Declared column types, doubling as the ColumnInspector used to infer
each materialized virtual column's output
// type (an expression's output type can depend on its input column types).
final RowSignature.Builder signatureBuilder = RowSignature.builder();
diff --git
a/processing/src/main/java/org/apache/druid/data/input/impl/TableProjectionSpec.java
b/processing/src/main/java/org/apache/druid/data/input/impl/TableProjectionSpec.java
new file mode 100644
index 00000000000..18545e45736
--- /dev/null
+++
b/processing/src/main/java/org/apache/druid/data/input/impl/TableProjectionSpec.java
@@ -0,0 +1,360 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.data.input.impl;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonTypeName;
+import com.google.common.collect.Sets;
+import org.apache.druid.error.InvalidInput;
+import org.apache.druid.java.util.common.granularity.Granularities;
+import org.apache.druid.java.util.common.granularity.Granularity;
+import org.apache.druid.query.OrderBy;
+import org.apache.druid.query.aggregation.AggregatorFactory;
+import org.apache.druid.segment.VirtualColumn;
+import org.apache.druid.segment.VirtualColumns;
+import org.apache.druid.segment.column.ColumnHolder;
+
+import javax.annotation.Nullable;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Objects;
+import java.util.Set;
+
+/**
+ * {@link BaseTableProjectionSpec} for a plain (non-clustered, non-rollup)
table: the operator declares a single
+ * ordered {@link #columns} list, the full set of columns in segment order,
and rows are stored and sorted by every
+ * column in declared order. The time position is an explicit positional entry
in {@link #columns} named
+ * {@code __time}. A plain base table is never rollup and has no metric
columns.
+ * <p>
+ * Operator-facing counterpart of the segment metadata-side {@link
org.apache.druid.segment.projections.TableProjectionSchema}.
+ * <p>
+ * {@link #getDimensionsSpec()} returns the unified spec built from {@link
#columns} in declared order with
+ * {@code forceSegmentSortByTime=false}; {@link #getOrdering()} is computed as
every column of {@link #columns}
+ * ascending, in list order.
+ * <p>
+ * Query granularity, when wanted, is a virtual column in {@link
#getVirtualColumns()} named
+ * {@link Granularities#GRANULARITY_VIRTUAL_COLUMN_NAME}. It is a granularity
<em>carrier</em>: it supplies the
+ * granularity that floors the stored {@code __time} column, and is NOT itself
a stored column, so it never appears in
+ * {@link #columns} (declare {@code __time} there as the time column). Absent
that virtual column the query granularity
+ * is {@code NONE}.
+ * <p>
+ * The granularity carrier is the only virtual column a plain base table
accepts. The standard segment-generation path
+ * this spec lowers into does not evaluate spec virtual columns (only the
clustered write path materializes them), so a
+ * materialized virtual column here would be silently dropped; computed
columns belong in a
+ * {@code transformSpec} instead. This can be relaxed if the plain write path
learns to materialize virtual columns.
+ */
+@JsonTypeName(TableProjectionSpec.TYPE_NAME)
+public final class TableProjectionSpec implements BaseTableProjectionSpec
+{
+ public static final String TYPE_NAME = "table";
+
+ private final VirtualColumns virtualColumns;
+ private final List<DimensionSchema> columns;
+ private final DimensionsSpec dimensionsSpec;
+ private final List<OrderBy> ordering;
+
+ public static Builder builder()
+ {
+ return new Builder();
+ }
+
+ @JsonCreator
+ public TableProjectionSpec(
+ @JsonProperty("virtualColumns") @Nullable VirtualColumns virtualColumns,
+ @JsonProperty("columns") List<DimensionSchema> columns
+ )
+ {
+ validate(columns);
+ this.virtualColumns = virtualColumns == null ? VirtualColumns.EMPTY :
virtualColumns;
+ validateVirtualColumns(this.virtualColumns);
+ this.columns = Collections.unmodifiableList(new ArrayList<>(columns));
+ this.dimensionsSpec = DimensionsSpec.builder()
+ .setDimensions(this.columns)
+ .setForceSegmentSortByTime(false)
+ .build();
+ this.ordering = computeOrdering(this.columns);
+ }
+
+ @Override
+ @JsonProperty
+ @JsonInclude(JsonInclude.Include.NON_DEFAULT)
+ public VirtualColumns getVirtualColumns()
+ {
+ return virtualColumns;
+ }
+
+ /**
+ * The full, ordered list of columns in segment order, with the explicit
{@code __time} (or query-granularity)
+ * marker at its declared position.
+ */
+ @JsonProperty("columns")
+ public List<DimensionSchema> getColumns()
+ {
+ return columns;
+ }
+
+ @Override
+ public AggregatorFactory[] getMetrics()
+ {
+ return new AggregatorFactory[0];
+ }
+
+ @Override
+ @JsonIgnore
+ public List<OrderBy> getOrdering()
+ {
+ return ordering;
+ }
+
+ @Override
+ @JsonIgnore
+ public DimensionsSpec getDimensionsSpec()
+ {
+ return dimensionsSpec;
+ }
+
+ /**
+ * Returns a copy of this spec with a new {@code queryGranularity},
expressed as a
+ * {@link Granularities#GRANULARITY_VIRTUAL_COLUMN_NAME} virtual column
added to {@link #getVirtualColumns()}. A
+ * {@code null} or {@code NONE} granularity is a no-op (no flooring), so
this returns {@code this} unchanged.
+ * <p>
+ * {@code ALL} is rejected: it has no granularity virtual column
representation, so accepting it would silently
+ * degrade to {@code NONE} when the spec is read back.
+ * <p>
+ * Idempotent: if the spec already declares a query-granularity virtual
column, that one is authoritative and this is
+ * a no-op. (The compaction path attaches the virtual column up front; the
MSQ generation path then calls this again
+ * with the query-derived granularity, which must not double-add.)
+ */
+ @Override
+ public TableProjectionSpec withQueryGranularity(@Nullable Granularity
queryGranularity)
+ {
+ if (Granularities.ALL.equals(queryGranularity)) {
+ throw InvalidInput.exception(
+ "Query granularity[ALL] is not supported for [%s] base tables",
+ TYPE_NAME
+ );
+ }
+ if (queryGranularity == null
+ || Granularities.NONE.equals(queryGranularity)
+ ||
virtualColumns.getVirtualColumn(Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME)
!= null) {
+ return this;
+ }
+ BaseTableProjectionSpec.validateQueryGranularity(queryGranularity,
TYPE_NAME);
+ final VirtualColumn granularityVirtualColumn =
+ Granularities.toVirtualColumn(queryGranularity,
Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME);
+ final List<VirtualColumn> merged = new
ArrayList<>(Arrays.asList(virtualColumns.getVirtualColumns()));
+ merged.add(granularityVirtualColumn);
+ return new TableProjectionSpec(VirtualColumns.create(merged), columns);
+ }
+
+ /**
+ * Compares table-spec state for compaction up-to-date checks: query
granularity is compared separately (via its
+ * carrier virtual column), so it is stripped from both sides; everything
else (the columns, and any other virtual
+ * columns) must match. A spec of a different type is never equivalent.
+ */
+ @Override
+ public boolean hasEqualCompactionState(BaseTableProjectionSpec other)
+ {
+ if (!(other instanceof TableProjectionSpec)) {
+ return false;
+ }
+ return withoutQueryGranularity().equals(((TableProjectionSpec)
other).withoutQueryGranularity());
+ }
+
+ @Override
+ public TableProjectionSpec withAdditionalColumns(@Nullable
List<DimensionSchema> additionalColumns)
+ {
+ if (additionalColumns == null || additionalColumns.isEmpty()) {
+ return this;
+ }
+ final List<DimensionSchema> revised = new ArrayList<>(columns.size() +
additionalColumns.size());
+ revised.addAll(columns);
+ for (DimensionSchema additionalColumn : additionalColumns) {
+ if (ColumnHolder.TIME_COLUMN_NAME.equals(additionalColumn.getName())) {
+ throw InvalidInput.exception(
+ "Cannot append column [%s] to a [%s] base table; it must be
declared at its position in the column list",
+ ColumnHolder.TIME_COLUMN_NAME,
+ TYPE_NAME
+ );
+ }
+ revised.add(additionalColumn);
+ }
+ // Duplicates of a declared column, and a column named for the
query-granularity carrier, are rejected by the
+ // constructor's validation.
+ return new TableProjectionSpec(virtualColumns, revised);
+ }
+
+ /**
+ * Returns a copy of this spec with the {@link
Granularities#GRANULARITY_VIRTUAL_COLUMN_NAME} virtual column removed,
+ * the inverse of {@link #withQueryGranularity(Granularity)}. If no such
virtual column is present this returns
+ * {@code this} unchanged. Used to compare schema independently of query
granularity in {@link #hasEqualCompactionState}.
+ */
+ private TableProjectionSpec withoutQueryGranularity()
+ {
+ if
(virtualColumns.getVirtualColumn(Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME)
== null) {
+ return this;
+ }
+ final List<VirtualColumn> remaining = new ArrayList<>();
+ for (VirtualColumn vc : virtualColumns.getVirtualColumns()) {
+ if
(!Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME.equals(vc.getOutputName())) {
+ remaining.add(vc);
+ }
+ }
+ return new TableProjectionSpec(VirtualColumns.create(remaining), columns);
+ }
+
+ private static void validate(List<DimensionSchema> columns)
+ {
+ if (columns == null || columns.isEmpty()) {
+ throw InvalidInput.exception("columns must be non-empty for [%s] base
table", TYPE_NAME);
+ }
+ final Set<String> seen = Sets.newHashSetWithExpectedSize(columns.size());
+ for (DimensionSchema d : columns) {
+ if (!seen.add(d.getName())) {
+ throw InvalidInput.exception("columns contains duplicate name [%s]",
d.getName());
+ }
+ }
+ boolean foundTime = false;
+ for (DimensionSchema column : columns) {
+ final String name = column.getName();
+ // The query-granularity virtual column is a granularity carrier in
virtualColumns (it floors the stored __time
+ // column); it is not itself a stored column, so it must not be declared
in 'columns'.
+ if (Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME.equals(name)) {
+ throw InvalidInput.exception(
+ "[%s] is the query-granularity virtual column, not a stored
column; declare it in 'virtualColumns' and use"
+ + " [%s] as the time column in 'columns'",
+ Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME,
+ ColumnHolder.TIME_COLUMN_NAME
+ );
+ }
+ if (ColumnHolder.TIME_COLUMN_NAME.equals(name)) {
+ foundTime = true;
+ }
+ }
+ if (!foundTime) {
+ throw InvalidInput.exception(
+ "[%s] base table must include [%s] in 'columns' to define the time
position",
+ TYPE_NAME,
+ ColumnHolder.TIME_COLUMN_NAME
+ );
+ }
+ }
+
+ /**
+ * A plain base table accepts only the {@link
Granularities#GRANULARITY_VIRTUAL_COLUMN_NAME} carrier: the standard
+ * segment-generation path does not evaluate spec virtual columns, so any
other virtual column would be dead metadata
+ * whose column never gets materialized. Computed columns belong in a {@code
transformSpec}.
+ */
+ private static void validateVirtualColumns(VirtualColumns virtualColumns)
+ {
+ for (VirtualColumn virtualColumn : virtualColumns.getVirtualColumns()) {
+ if
(!Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME.equals(virtualColumn.getOutputName()))
{
+ throw InvalidInput.exception(
+ "virtual column [%s] is not supported: a [%s] base table stores
only declared columns and does not"
+ + " materialize virtual columns; use a transformSpec to compute
columns at ingest, or the [%s] virtual"
+ + " column to carry query granularity",
+ virtualColumn.getOutputName(),
+ TYPE_NAME,
+ Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME
+ );
+ }
+ BaseTableProjectionSpec.validateGranularity(virtualColumn, TYPE_NAME);
+ }
+ }
+
+ private static List<OrderBy> computeOrdering(List<DimensionSchema> columns)
+ {
+ final List<OrderBy> ordering = new ArrayList<>(columns.size());
+ for (DimensionSchema d : columns) {
+ ordering.add(OrderBy.ascending(d.getName()));
+ }
+ return Collections.unmodifiableList(ordering);
+ }
+
+ @Override
+ public boolean equals(Object o)
+ {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ TableProjectionSpec that = (TableProjectionSpec) o;
+ return Objects.equals(virtualColumns, that.virtualColumns)
+ && Objects.equals(columns, that.columns);
+ }
+
+ @Override
+ public int hashCode()
+ {
+ return Objects.hash(virtualColumns, columns);
+ }
+
+ @Override
+ public String toString()
+ {
+ return "TableProjectionSpec{" +
+ "virtualColumns=" + virtualColumns +
+ ", columns=" + columns +
+ '}';
+ }
+
+ /**
+ * Fluent builder for {@link TableProjectionSpec}, avoiding the
constructor's positional nullable leading
+ * {@code virtualColumns} argument. {@link #columns} (the full ordered
column list) is required.
+ */
+ public static final class Builder
+ {
+ @Nullable
+ private VirtualColumns virtualColumns;
+ private List<DimensionSchema> columns = Collections.emptyList();
+
+ public Builder virtualColumns(@Nullable VirtualColumns virtualColumns)
+ {
+ this.virtualColumns = virtualColumns;
+ return this;
+ }
+
+ /**
+ * The full, ordered list of columns in segment order, including the
explicit time marker.
+ */
+ public Builder columns(List<DimensionSchema> columns)
+ {
+ this.columns = columns;
+ return this;
+ }
+
+ public Builder columns(DimensionSchema... columns)
+ {
+ return columns(Arrays.asList(columns));
+ }
+
+ public TableProjectionSpec build()
+ {
+ return new TableProjectionSpec(virtualColumns, columns);
+ }
+ }
+}
diff --git
a/processing/src/main/java/org/apache/druid/java/util/common/granularity/Granularities.java
b/processing/src/main/java/org/apache/druid/java/util/common/granularity/Granularities.java
index 0bba55f9693..803eb4f59b0 100644
---
a/processing/src/main/java/org/apache/druid/java/util/common/granularity/Granularities.java
+++
b/processing/src/main/java/org/apache/druid/java/util/common/granularity/Granularities.java
@@ -37,6 +37,7 @@ import org.apache.druid.segment.column.ColumnHolder;
import org.apache.druid.segment.column.ColumnType;
import org.apache.druid.segment.projections.AggregateProjectionSchema;
import org.apache.druid.segment.virtual.ExpressionVirtualColumn;
+import org.joda.time.DateTimeZone;
import javax.annotation.Nullable;
import java.util.Arrays;
@@ -183,7 +184,9 @@ public class Granularities
* IMPORTANT - this method DOES NOT VERIFY that the virtual column has a
single input that is a time column
* ({@link ColumnHolder#TIME_COLUMN_NAME} or equivalent projection time
column as defined by
* {@link AggregateProjectionSchema#getTimeColumnName()}). Callers must
verify this externally before
- * calling this method by examining {@link VirtualColumn#requiredColumns()}.
+ * calling this method by examining {@link VirtualColumn#requiredColumns()},
or call
+ * {@link #fromTimeVirtualColumn(VirtualColumn)}, which performs the {@link
ColumnHolder#TIME_COLUMN_NAME} check
+ * itself.
* <p>
* This method also does not handle other time expressions, or if the
virtual column is just an identifier for a
* time column
@@ -215,4 +218,30 @@ public class Granularities
}
return null;
}
+
+ /**
+ * Like {@link #fromVirtualColumn(VirtualColumn)}, but first verifies that
the virtual column reads
+ * {@link ColumnHolder#TIME_COLUMN_NAME} and nothing else, returning null
otherwise.
+ */
+ @Nullable
+ public static Granularity fromTimeVirtualColumn(@Nullable VirtualColumn
virtualColumn)
+ {
+ if (virtualColumn == null
+ ||
!Collections.singletonList(ColumnHolder.TIME_COLUMN_NAME).equals(virtualColumn.requiredColumns()))
{
+ return null;
+ }
+ return fromVirtualColumn(virtualColumn);
+ }
+
+ /**
+ * Returns true when {@code granularity} is exactly a {@link
PeriodGranularity} in the UTC time zone with no origin.
+ */
+ public static boolean isStandardUtcPeriod(@Nullable Granularity granularity)
+ {
+ if (granularity == null ||
!granularity.getClass().equals(PeriodGranularity.class)) {
+ return false;
+ }
+ final PeriodGranularity period = (PeriodGranularity) granularity;
+ return DateTimeZone.UTC.equals(period.getTimeZone()) && period.getOrigin()
== null;
+ }
}
diff --git
a/processing/src/main/java/org/apache/druid/segment/VirtualColumn.java
b/processing/src/main/java/org/apache/druid/segment/VirtualColumn.java
index 3c2fdcf5f69..ee26bacde57 100644
--- a/processing/src/main/java/org/apache/druid/segment/VirtualColumn.java
+++ b/processing/src/main/java/org/apache/druid/segment/VirtualColumn.java
@@ -548,6 +548,26 @@ public interface VirtualColumn extends Cacheable
return null;
}
+ /**
+ * Returns true if the virtual column supports {@link
#withOutputName(String)}
+ */
+ default boolean supportsOutputNameRewrite()
+ {
+ return false;
+ }
+
+ /**
+ * Returns a copy of this virtual column identical except for the given
output name. Used where a virtual column
+ * generated under a synthetic name (such as the planner's {@code v0},
{@code v1}, ...) must be re-issued under a
+ * declared column name — for example, lifting the computed columns of a
catalog base-table definition out of a
+ * planned query. Callers should check {@link #supportsOutputNameRewrite()}
first to ensure that this method is
+ * supported.
+ */
+ default VirtualColumn withOutputName(String outputName)
+ {
+ throw DruidException.defensive("Output name rewrite is not supported by
virtual column[%s].", getOutputName());
+ }
+
@SubclassesMustOverrideEqualsAndHashCode
interface EquivalenceKey
{
diff --git
a/processing/src/main/java/org/apache/druid/segment/virtual/ExpressionVirtualColumn.java
b/processing/src/main/java/org/apache/druid/segment/virtual/ExpressionVirtualColumn.java
index 1ea737b9d35..efee2eaa6b3 100644
---
a/processing/src/main/java/org/apache/druid/segment/virtual/ExpressionVirtualColumn.java
+++
b/processing/src/main/java/org/apache/druid/segment/virtual/ExpressionVirtualColumn.java
@@ -154,6 +154,18 @@ public class ExpressionVirtualColumn implements
VirtualColumn
return expression.parsed;
}
+ @Override
+ public boolean supportsOutputNameRewrite()
+ {
+ return true;
+ }
+
+ @Override
+ public ExpressionVirtualColumn withOutputName(final String outputName)
+ {
+ return new ExpressionVirtualColumn(outputName, getExpression(),
getOutputType(), expression.parsed);
+ }
+
@Override
public DimensionSelector makeDimensionSelector(
final DimensionSpec dimensionSpec,
diff --git
a/processing/src/main/java/org/apache/druid/segment/virtual/NestedFieldVirtualColumn.java
b/processing/src/main/java/org/apache/druid/segment/virtual/NestedFieldVirtualColumn.java
index 1cbcf36871f..f067a222bae 100644
---
a/processing/src/main/java/org/apache/druid/segment/virtual/NestedFieldVirtualColumn.java
+++
b/processing/src/main/java/org/apache/druid/segment/virtual/NestedFieldVirtualColumn.java
@@ -233,6 +233,26 @@ public class NestedFieldVirtualColumn implements
VirtualColumn
return fieldSpec.processFromRaw;
}
+ @Override
+ public boolean supportsOutputNameRewrite()
+ {
+ return true;
+ }
+
+ @Override
+ public NestedFieldVirtualColumn withOutputName(String newOutputName)
+ {
+ return new NestedFieldVirtualColumn(
+ getColumnName(),
+ newOutputName,
+ getExpectedType(),
+ getPathParts(),
+ isProcessFromRaw(),
+ null,
+ null
+ );
+ }
+
@Override
public DimensionSelector makeDimensionSelector(
DimensionSpec dimensionSpec,
diff --git
a/processing/src/main/java/org/apache/druid/segment/virtual/NestedMergeVirtualColumn.java
b/processing/src/main/java/org/apache/druid/segment/virtual/NestedMergeVirtualColumn.java
index 7c46a0bd2ef..7946a0d37ed 100644
---
a/processing/src/main/java/org/apache/druid/segment/virtual/NestedMergeVirtualColumn.java
+++
b/processing/src/main/java/org/apache/druid/segment/virtual/NestedMergeVirtualColumn.java
@@ -141,6 +141,12 @@ public class NestedMergeVirtualColumn extends
SpecializedExpressionVirtualColumn
);
}
+ @Override
+ public NestedMergeVirtualColumn withOutputName(String outputName)
+ {
+ return new NestedMergeVirtualColumn(outputName, columns, macroTable);
+ }
+
@Override
public String toString()
{
diff --git
a/processing/src/main/java/org/apache/druid/segment/virtual/NestedObjectVirtualColumn.java
b/processing/src/main/java/org/apache/druid/segment/virtual/NestedObjectVirtualColumn.java
index 8096b26490f..b9e3cb07c0b 100644
---
a/processing/src/main/java/org/apache/druid/segment/virtual/NestedObjectVirtualColumn.java
+++
b/processing/src/main/java/org/apache/druid/segment/virtual/NestedObjectVirtualColumn.java
@@ -158,6 +158,12 @@ public class NestedObjectVirtualColumn extends
SpecializedExpressionVirtualColum
);
}
+ @Override
+ public NestedObjectVirtualColumn withOutputName(String outputName)
+ {
+ return new NestedObjectVirtualColumn(outputName, keyExprMap, macroTable);
+ }
+
@Override
public String toString()
{
diff --git
a/processing/src/main/java/org/apache/druid/segment/virtual/SpecializedExpressionVirtualColumn.java
b/processing/src/main/java/org/apache/druid/segment/virtual/SpecializedExpressionVirtualColumn.java
index b08ee20888d..f0ad5e85814 100644
---
a/processing/src/main/java/org/apache/druid/segment/virtual/SpecializedExpressionVirtualColumn.java
+++
b/processing/src/main/java/org/apache/druid/segment/virtual/SpecializedExpressionVirtualColumn.java
@@ -196,6 +196,19 @@ public abstract class SpecializedExpressionVirtualColumn
implements VirtualColum
return delegate.getEquivalanceKey();
}
+ @Override
+ public boolean supportsOutputNameRewrite()
+ {
+ return true;
+ }
+
+ /**
+ * Declared abstract rather than inherited so that every specialization
implements the rename that
+ * {@link #supportsOutputNameRewrite()} advertises on its behalf.
+ */
+ @Override
+ public abstract SpecializedExpressionVirtualColumn withOutputName(String
outputName);
+
@Override
public SelectableColumn toSelectableColumn(ColumnIndexSelector
columnSelector)
{
diff --git
a/processing/src/test/java/org/apache/druid/data/input/impl/AggregateProjectionSpecTest.java
b/processing/src/test/java/org/apache/druid/data/input/impl/AggregateProjectionSpecTest.java
index 638b7029ce9..a5c67e9c83f 100644
---
a/processing/src/test/java/org/apache/druid/data/input/impl/AggregateProjectionSpecTest.java
+++
b/processing/src/test/java/org/apache/druid/data/input/impl/AggregateProjectionSpecTest.java
@@ -184,6 +184,27 @@ class AggregateProjectionSpecTest extends
InitializedNullHandlingTest
).toMetadataSchema().getTimeColumnName());
}
+ @Test
+ void testComputeOrdering_floorOfNonTimeColumnIsNotATimeColumn()
+ {
+ // A grouping column flooring some column other than {@code __time} is a
perfectly good grouping column, but it is
+ // not a time grouping: its granularity describes buckets of the other
column, so designating it the time column
+ // would claim a time bucketing the projection does not have.
+ ExpressionVirtualColumn dailyOther = new ExpressionVirtualColumn(
+ "dailyOther",
+ "timestamp_floor(other, 'P1D', null, null)",
+ ColumnType.LONG,
+ TestExprMacroTable.INSTANCE
+ );
+ Assertions.assertNull(new AggregateProjectionSpec(
+ "some_projection",
+ null,
+ VirtualColumns.create(dailyOther),
+ List.of(new LongDimensionSchema("dailyOther")),
+ new AggregatorFactory[]{}
+ ).toMetadataSchema().getTimeColumnName());
+ }
+
@Test
void testMissingName()
{
diff --git
a/processing/src/test/java/org/apache/druid/data/input/impl/ClusteredValueGroupsBaseTableProjectionSpecTest.java
b/processing/src/test/java/org/apache/druid/data/input/impl/ClusteredValueGroupsBaseTableProjectionSpecTest.java
index a09fdfc42de..bb0f7226fd0 100644
---
a/processing/src/test/java/org/apache/druid/data/input/impl/ClusteredValueGroupsBaseTableProjectionSpecTest.java
+++
b/processing/src/test/java/org/apache/druid/data/input/impl/ClusteredValueGroupsBaseTableProjectionSpecTest.java
@@ -21,7 +21,10 @@ package org.apache.druid.data.input.impl;
import com.google.common.collect.ImmutableList;
import org.apache.druid.error.DruidException;
+import org.apache.druid.java.util.common.DateTimes;
+import org.apache.druid.java.util.common.granularity.DurationGranularity;
import org.apache.druid.java.util.common.granularity.Granularities;
+import org.apache.druid.java.util.common.granularity.PeriodGranularity;
import org.apache.druid.query.OrderBy;
import org.apache.druid.query.dimension.DimensionSpec;
import org.apache.druid.query.expression.TestExprMacroTable;
@@ -34,6 +37,7 @@ import org.apache.druid.segment.column.ColumnCapabilities;
import org.apache.druid.segment.column.ColumnType;
import org.apache.druid.segment.virtual.ExpressionVirtualColumn;
import org.apache.druid.testing.InitializedNullHandlingTest;
+import org.joda.time.Period;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
@@ -89,6 +93,107 @@ class ClusteredValueGroupsBaseTableProjectionSpecTest
extends InitializedNullHan
Assertions.assertTrue(e.getMessage().contains("ALL"));
}
+ @Test
+ void testWithQueryGranularityDurationIsRejected()
+ {
+ final DruidException e = Assertions.assertThrows(
+ DruidException.class,
+ () -> tenantSpec().withQueryGranularity(new
DurationGranularity(3_600_000, null))
+ );
+ Assertions.assertTrue(e.getMessage().contains("only period granularities
in the UTC time zone"), e.getMessage());
+ }
+
+ @Test
+ void testWithQueryGranularityNonUtcPeriodIsRejected()
+ {
+ final DruidException e = Assertions.assertThrows(
+ DruidException.class,
+ () -> tenantSpec().withQueryGranularity(
+ new PeriodGranularity(new Period("P1D"), null,
DateTimes.inferTzFromString("America/Los_Angeles"))
+ )
+ );
+ Assertions.assertTrue(e.getMessage().contains("only period granularities
in the UTC time zone"), e.getMessage());
+ }
+
+ @Test
+ void testWithQueryGranularityOriginPeriodIsRejected()
+ {
+ // An origin shifts the bucket boundaries, which the projection machinery
does not treat as a standard time
+ // grouping (AggregateProjectionSpec requires a null origin to recognize a
time column), so it is rejected here
+ // for the same reason as a non-UTC time zone.
+ final DruidException e = Assertions.assertThrows(
+ DruidException.class,
+ () -> tenantSpec().withQueryGranularity(
+ new PeriodGranularity(new Period("P1D"),
DateTimes.of("2020-01-02T03:00:00Z"), null)
+ )
+ );
+ Assertions.assertTrue(
+ e.getMessage().contains("only period granularities in the UTC time
zone without an origin"),
+ e.getMessage()
+ );
+ }
+
+ @Test
+ void testNonUtcGranularityCarrierRejectedAtConstruction()
+ {
+ final DruidException e = Assertions.assertThrows(
+ DruidException.class,
+ () -> ClusteredValueGroupsBaseTableProjectionSpec.builder()
+ .virtualColumns(VirtualColumns.create(
+ Granularities.toVirtualColumn(
+ new PeriodGranularity(new Period("P1D"), null,
DateTimes.inferTzFromString("America/Los_Angeles")),
+ Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME
+ )
+ ))
+ .columns(new StringDimensionSchema("tenant"), new
LongDimensionSchema("__time"))
+ .clusteringColumns("tenant")
+ .build()
+ );
+ Assertions.assertTrue(e.getMessage().contains("only period granularities
in the UTC time zone"), e.getMessage());
+ }
+
+ @Test
+ void testUndecodableGranularityCarrierRejected()
+ {
+ final DruidException e = Assertions.assertThrows(
+ DruidException.class,
+ () -> ClusteredValueGroupsBaseTableProjectionSpec.builder()
+ .virtualColumns(VirtualColumns.create(
+ new ExpressionVirtualColumn(
+ Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME,
+ "__time",
+ ColumnType.LONG,
+ TestExprMacroTable.INSTANCE
+ )
+ ))
+ .columns(new StringDimensionSchema("tenant"), new
LongDimensionSchema("__time"))
+ .clusteringColumns("tenant")
+ .build()
+ );
+ Assertions.assertTrue(e.getMessage().contains("does not encode a query
granularity"), e.getMessage());
+ }
+
+ @Test
+ void testGranularityCarrierReadingOtherColumnRejected()
+ {
+ final DruidException e = Assertions.assertThrows(
+ DruidException.class,
+ () -> ClusteredValueGroupsBaseTableProjectionSpec.builder()
+ .virtualColumns(VirtualColumns.create(
+ new ExpressionVirtualColumn(
+ Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME,
+ "timestamp_floor(\"tenant\", 'P1D')",
+ ColumnType.LONG,
+ TestExprMacroTable.INSTANCE
+ )
+ ))
+ .columns(new StringDimensionSchema("tenant"), new
LongDimensionSchema("__time"))
+ .clusteringColumns("tenant")
+ .build()
+ );
+ Assertions.assertTrue(e.getMessage().contains("must be computed from
[__time] alone"), e.getMessage());
+ }
+
@Test
void testGetQueryGranularityRoundTrips()
{
diff --git
a/processing/src/test/java/org/apache/druid/data/input/impl/TableProjectionSpecTest.java
b/processing/src/test/java/org/apache/druid/data/input/impl/TableProjectionSpecTest.java
new file mode 100644
index 00000000000..459e663dca4
--- /dev/null
+++
b/processing/src/test/java/org/apache/druid/data/input/impl/TableProjectionSpecTest.java
@@ -0,0 +1,428 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.data.input.impl;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.collect.ImmutableList;
+import org.apache.druid.error.DruidException;
+import org.apache.druid.java.util.common.DateTimes;
+import org.apache.druid.java.util.common.granularity.DurationGranularity;
+import org.apache.druid.java.util.common.granularity.Granularities;
+import org.apache.druid.java.util.common.granularity.PeriodGranularity;
+import org.apache.druid.query.OrderBy;
+import org.apache.druid.query.expression.TestExprMacroTable;
+import org.apache.druid.segment.TestHelper;
+import org.apache.druid.segment.VirtualColumn;
+import org.apache.druid.segment.VirtualColumns;
+import org.apache.druid.segment.column.ColumnType;
+import org.apache.druid.segment.virtual.ExpressionVirtualColumn;
+import org.apache.druid.testing.InitializedNullHandlingTest;
+import org.joda.time.Period;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+import java.util.Collections;
+import java.util.stream.Collectors;
+
+class TableProjectionSpecTest extends InitializedNullHandlingTest
+{
+ private static TableProjectionSpec pagesSpec()
+ {
+ return TableProjectionSpec.builder()
+ .columns(new StringDimensionSchema("page"), new
LongDimensionSchema("__time"), new LongDimensionSchema("cnt"))
+ .build();
+ }
+
+ @Test
+ void testOrderingAndDimensionsSpecFollowDeclaredOrder()
+ {
+ final TableProjectionSpec spec = pagesSpec();
+
+ // Declared order is both storage and sort order: every column ascending,
in list order, with __time at its
+ // declared position rather than forced first.
+ Assertions.assertEquals(
+ ImmutableList.of(OrderBy.ascending("page"),
OrderBy.ascending("__time"), OrderBy.ascending("cnt")),
+ spec.getOrdering()
+ );
+ Assertions.assertEquals(spec.getColumns(),
spec.getDimensionsSpec().getDimensions());
+
Assertions.assertFalse(spec.getDimensionsSpec().isForceSegmentSortByTime());
+ Assertions.assertEquals(0, spec.getMetrics().length);
+ }
+
+ @Test
+ void testWithQueryGranularityAddsVirtualGranularityColumn()
+ {
+ final TableProjectionSpec spec =
pagesSpec().withQueryGranularity(Granularities.HOUR);
+
+ final VirtualColumn vc =
spec.getVirtualColumns().getVirtualColumn(Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME);
+ Assertions.assertNotNull(vc);
+ Assertions.assertEquals(Granularities.HOUR,
Granularities.fromVirtualColumn(vc));
+
+ // The rest of the spec is unchanged.
+ Assertions.assertEquals(pagesSpec().getColumns(), spec.getColumns());
+ Assertions.assertEquals(pagesSpec().getOrdering(), spec.getOrdering());
+ Assertions.assertArrayEquals(pagesSpec().getMetrics(), spec.getMetrics());
+ }
+
+ @Test
+ void testWithQueryGranularityNullAndNoneAreNoOps()
+ {
+ // Absent __virtualGranularity virtual column already means NONE, so
null/NONE add nothing and return the same spec.
+ final TableProjectionSpec spec = pagesSpec();
+ Assertions.assertSame(spec, spec.withQueryGranularity(null));
+ Assertions.assertSame(spec, spec.withQueryGranularity(Granularities.NONE));
+ }
+
+ @Test
+ void testWithQueryGranularityAllIsRejected()
+ {
+ // ALL has no granularity virtual column representation, so accepting it
would silently degrade to NONE when the
+ // spec is read back; it is rejected rather than silently ignored.
+ final TableProjectionSpec spec = pagesSpec();
+ final DruidException e = Assertions.assertThrows(
+ DruidException.class,
+ () -> spec.withQueryGranularity(Granularities.ALL)
+ );
+ Assertions.assertTrue(e.getMessage().contains("ALL"));
+ }
+
+ @Test
+ void testWithQueryGranularityDurationIsRejected()
+ {
+ final DruidException e = Assertions.assertThrows(
+ DruidException.class,
+ () -> pagesSpec().withQueryGranularity(new
DurationGranularity(3_600_000, null))
+ );
+ Assertions.assertTrue(e.getMessage().contains("only period granularities
in the UTC time zone"), e.getMessage());
+ }
+
+ @Test
+ void testWithQueryGranularityNonUtcPeriodIsRejected()
+ {
+ final DruidException e = Assertions.assertThrows(
+ DruidException.class,
+ () -> pagesSpec().withQueryGranularity(
+ new PeriodGranularity(new Period("P1D"), null,
DateTimes.inferTzFromString("America/Los_Angeles"))
+ )
+ );
+ Assertions.assertTrue(e.getMessage().contains("only period granularities
in the UTC time zone"), e.getMessage());
+ }
+
+ @Test
+ void testWithQueryGranularityOriginPeriodIsRejected()
+ {
+ // An origin shifts the bucket boundaries, which the projection machinery
does not treat as a standard time
+ // grouping (AggregateProjectionSpec requires a null origin to recognize a
time column), so it is rejected here
+ // for the same reason as a non-UTC time zone.
+ final DruidException e = Assertions.assertThrows(
+ DruidException.class,
+ () -> pagesSpec().withQueryGranularity(
+ new PeriodGranularity(new Period("P1D"),
DateTimes.of("2020-01-02T03:00:00Z"), null)
+ )
+ );
+ Assertions.assertTrue(
+ e.getMessage().contains("only period granularities in the UTC time
zone without an origin"),
+ e.getMessage()
+ );
+ }
+
+ @Test
+ void testNonUtcGranularityCarrierRejectedAtConstruction()
+ {
+ final DruidException e = Assertions.assertThrows(
+ DruidException.class,
+ () -> TableProjectionSpec.builder()
+ .virtualColumns(VirtualColumns.create(
+ Granularities.toVirtualColumn(
+ new PeriodGranularity(new Period("P1D"), null,
DateTimes.inferTzFromString("America/Los_Angeles")),
+ Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME
+ )
+ ))
+ .columns(new StringDimensionSchema("page"), new
LongDimensionSchema("__time"))
+ .build()
+ );
+ Assertions.assertTrue(e.getMessage().contains("only period granularities
in the UTC time zone"), e.getMessage());
+ }
+
+ @Test
+ void testGranularityCarrierReadingOtherColumnRejected()
+ {
+ final DruidException e = Assertions.assertThrows(
+ DruidException.class,
+ () -> TableProjectionSpec.builder()
+ .virtualColumns(VirtualColumns.create(
+ new ExpressionVirtualColumn(
+ Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME,
+ "timestamp_floor(\"page\", 'P1D')",
+ ColumnType.LONG,
+ TestExprMacroTable.INSTANCE
+ )
+ ))
+ .columns(new StringDimensionSchema("page"), new
LongDimensionSchema("__time"))
+ .build()
+ );
+ Assertions.assertTrue(e.getMessage().contains("must be computed from
[__time] alone"), e.getMessage());
+ }
+
+ @Test
+ void testUndecodableGranularityCarrierRejected()
+ {
+ final DruidException e = Assertions.assertThrows(
+ DruidException.class,
+ () -> TableProjectionSpec.builder()
+ .virtualColumns(VirtualColumns.create(
+ new ExpressionVirtualColumn(
+ Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME,
+ "__time",
+ ColumnType.LONG,
+ TestExprMacroTable.INSTANCE
+ )
+ ))
+ .columns(new StringDimensionSchema("page"), new
LongDimensionSchema("__time"))
+ .build()
+ );
+ Assertions.assertTrue(e.getMessage().contains("does not encode a query
granularity"), e.getMessage());
+ }
+
+ @Test
+ void testGetQueryGranularityRoundTrips()
+ {
+ Assertions.assertEquals(
+ Granularities.HOUR,
+
pagesSpec().withQueryGranularity(Granularities.HOUR).getQueryGranularity()
+ );
+ }
+
+ @Test
+ void testGetQueryGranularityIsNoneWhenNoVirtualColumn()
+ {
+ Assertions.assertEquals(Granularities.NONE,
pagesSpec().getQueryGranularity());
+ }
+
+ @Test
+ void testWithQueryGranularityIsIdempotentWhenAlreadyPresent()
+ {
+ // Once a query-granularity virtual column is present it is authoritative;
a second call is a no-op and does not
+ // double-add or change it (the compaction path attaches it up front, then
MSQ generation calls this again).
+ final TableProjectionSpec withGranularity =
pagesSpec().withQueryGranularity(Granularities.HOUR);
+ final TableProjectionSpec reapplied =
withGranularity.withQueryGranularity(Granularities.DAY);
+
+ Assertions.assertSame(withGranularity, reapplied);
+ Assertions.assertEquals(
+ Granularities.HOUR,
+ Granularities.fromVirtualColumn(
+
reapplied.getVirtualColumns().getVirtualColumn(Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME)
+ )
+ );
+ }
+
+ @Test
+ void testHasEqualCompactionStateIgnoresQueryGranularity()
+ {
+ // Two specs that differ only in their query granularity are equivalent
for compaction (query granularity is
+ // compared by its own check), regardless of which granularity each
carries.
+
Assertions.assertTrue(pagesSpec().withQueryGranularity(Granularities.HOUR).hasEqualCompactionState(pagesSpec()));
+
Assertions.assertTrue(pagesSpec().hasEqualCompactionState(pagesSpec().withQueryGranularity(Granularities.HOUR)));
+ Assertions.assertTrue(
+ pagesSpec().withQueryGranularity(Granularities.HOUR)
+
.hasEqualCompactionState(pagesSpec().withQueryGranularity(Granularities.DAY))
+ );
+ }
+
+ @Test
+ void testHasEqualCompactionStateComparesColumns()
+ {
+ final TableProjectionSpec differentColumns = TableProjectionSpec.builder()
+ .columns(new StringDimensionSchema("page"), new
LongDimensionSchema("__time"))
+ .build();
+
+
Assertions.assertFalse(pagesSpec().hasEqualCompactionState(differentColumns));
+ Assertions.assertTrue(pagesSpec().hasEqualCompactionState(pagesSpec()));
+ }
+
+ @Test
+ void testHasEqualCompactionStateRejectsDifferentSpecType()
+ {
+ // A clustered spec over the same columns is a different physical layout,
never equivalent to a plain table.
+ final ClusteredValueGroupsBaseTableProjectionSpec clustered =
ClusteredValueGroupsBaseTableProjectionSpec.builder()
+ .columns(new StringDimensionSchema("page"), new
LongDimensionSchema("__time"), new LongDimensionSchema("cnt"))
+ .clusteringColumns("page")
+ .build();
+
+ Assertions.assertFalse(pagesSpec().hasEqualCompactionState(clustered));
+ Assertions.assertFalse(clustered.hasEqualCompactionState(pagesSpec()));
+ }
+
+ @Test
+ void testWithAdditionalColumnsAppendsAfterDeclaredColumns()
+ {
+ final TableProjectionSpec spec = pagesSpec().withAdditionalColumns(
+ ImmutableList.of(new LongDimensionSchema("clicks"), new
StringDimensionSchema("city"))
+ );
+
+ Assertions.assertEquals(
+ ImmutableList.of("page", "__time", "cnt", "clicks", "city"),
+
spec.getColumns().stream().map(DimensionSchema::getName).collect(Collectors.toList())
+ );
+ // Rows are physically sorted by every column present, so the appended
columns join the ordering at the end.
+ Assertions.assertEquals(
+ ImmutableList.of(
+ OrderBy.ascending("page"),
+ OrderBy.ascending("__time"),
+ OrderBy.ascending("cnt"),
+ OrderBy.ascending("clicks"),
+ OrderBy.ascending("city")
+ ),
+ spec.getOrdering()
+ );
+ Assertions.assertEquals(spec.getColumns(),
spec.getDimensionsSpec().getDimensions());
+ }
+
+ @Test
+ void testWithAdditionalColumnsNullAndEmptyAreNoOps()
+ {
+ final TableProjectionSpec spec = pagesSpec();
+ Assertions.assertSame(spec, spec.withAdditionalColumns(null));
+ Assertions.assertSame(spec,
spec.withAdditionalColumns(Collections.emptyList()));
+ }
+
+ @Test
+ void testWithAdditionalColumnsKeepsQueryGranularity()
+ {
+ final TableProjectionSpec spec = pagesSpec()
+ .withQueryGranularity(Granularities.HOUR)
+ .withAdditionalColumns(ImmutableList.of(new
LongDimensionSchema("clicks")));
+
+ Assertions.assertEquals(Granularities.HOUR, spec.getQueryGranularity());
+ Assertions.assertEquals("clicks",
spec.getColumns().get(spec.getColumns().size() - 1).getName());
+ }
+
+ @Test
+ void testWithAdditionalColumnsRejectsTimeColumn()
+ {
+ // __time marks a position in the column list, so it can never arrive as
an appended extra.
+ final DruidException e = Assertions.assertThrows(
+ DruidException.class,
+ () -> pagesSpec().withAdditionalColumns(ImmutableList.of(new
LongDimensionSchema("__time")))
+ );
+ Assertions.assertTrue(e.getMessage().contains("[__time]"));
+ }
+
+ @Test
+ void testWithAdditionalColumnsRejectsDuplicateOfDeclaredColumn()
+ {
+ final DruidException e = Assertions.assertThrows(
+ DruidException.class,
+ () -> pagesSpec().withAdditionalColumns(ImmutableList.of(new
StringDimensionSchema("page")))
+ );
+ Assertions.assertTrue(e.getMessage().contains("duplicate name [page]"));
+ }
+
+ @Test
+ void testEmptyColumnsRejected()
+ {
+ final DruidException e = Assertions.assertThrows(
+ DruidException.class,
+ () -> TableProjectionSpec.builder().build()
+ );
+ Assertions.assertTrue(e.getMessage().contains("non-empty"));
+ }
+
+ @Test
+ void testDuplicateColumnRejected()
+ {
+ final DruidException e = Assertions.assertThrows(
+ DruidException.class,
+ () -> TableProjectionSpec.builder()
+ .columns(
+ new StringDimensionSchema("page"),
+ new StringDimensionSchema("page"),
+ new LongDimensionSchema("__time")
+ )
+ .build()
+ );
+ Assertions.assertTrue(e.getMessage().contains("duplicate name [page]"));
+ }
+
+ @Test
+ void testMissingTimeColumnRejected()
+ {
+ final DruidException e = Assertions.assertThrows(
+ DruidException.class,
+ () -> TableProjectionSpec.builder().columns(new
StringDimensionSchema("page")).build()
+ );
+ Assertions.assertTrue(e.getMessage().contains("[__time]"));
+ }
+
+ @Test
+ void testGranularityCarrierInColumnsRejected()
+ {
+ // The query-granularity virtual column is a carrier in virtualColumns,
never a stored column.
+ final DruidException e = Assertions.assertThrows(
+ DruidException.class,
+ () -> TableProjectionSpec.builder()
+ .columns(
+ new StringDimensionSchema("page"),
+ new
LongDimensionSchema(Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME)
+ )
+ .build()
+ );
+
Assertions.assertTrue(e.getMessage().contains(Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME));
+ }
+
+ @Test
+ void testNonCarrierVirtualColumnRejected()
+ {
+ // The standard segment-generation path does not evaluate spec virtual
columns, so any virtual column other than
+ // the query-granularity carrier would never be materialized; computed
columns belong in a transformSpec.
+ final DruidException e = Assertions.assertThrows(
+ DruidException.class,
+ () -> TableProjectionSpec.builder()
+ .virtualColumns(VirtualColumns.create(
+ new ExpressionVirtualColumn("page_upper", "upper(page)",
ColumnType.STRING, TestExprMacroTable.INSTANCE)
+ ))
+ .columns(
+ new StringDimensionSchema("page"),
+ new StringDimensionSchema("page_upper"),
+ new LongDimensionSchema("__time")
+ )
+ .build()
+ );
+ Assertions.assertTrue(e.getMessage().contains("[page_upper]"));
+ Assertions.assertTrue(e.getMessage().contains("transformSpec"));
+ }
+
+ @Test
+ void testSerdeRoundTripsThroughInterface() throws Exception
+ {
+ final ObjectMapper mapper = TestHelper.makeJsonMapper();
+
+ final BaseTableProjectionSpec bare = pagesSpec();
+ final String bareJson = mapper.writeValueAsString(bare);
+ Assertions.assertTrue(bareJson.contains("\"type\":\"table\""), bareJson);
+ Assertions.assertEquals(bare, mapper.readValue(bareJson,
BaseTableProjectionSpec.class));
+
+ final BaseTableProjectionSpec withGranularity =
pagesSpec().withQueryGranularity(Granularities.HOUR);
+ Assertions.assertEquals(
+ withGranularity,
+ mapper.readValue(mapper.writeValueAsString(withGranularity),
BaseTableProjectionSpec.class)
+ );
+ }
+}
diff --git
a/processing/src/test/java/org/apache/druid/segment/virtual/ExpressionVirtualColumnTest.java
b/processing/src/test/java/org/apache/druid/segment/virtual/ExpressionVirtualColumnTest.java
index 558791c67ee..c0c5c500f81 100644
---
a/processing/src/test/java/org/apache/druid/segment/virtual/ExpressionVirtualColumnTest.java
+++
b/processing/src/test/java/org/apache/druid/segment/virtual/ExpressionVirtualColumnTest.java
@@ -794,6 +794,18 @@ public class ExpressionVirtualColumnTest extends
InitializedNullHandlingTest
Assertions.assertEquals(ImmutableList.of("a", "b", "c"),
multiConstantSelector.getObject());
}
+ @Test
+ public void testWithOutputName()
+ {
+ Assertions.assertTrue(X_PLUS_Y.supportsOutputNameRewrite());
+ final ExpressionVirtualColumn renamed =
X_PLUS_Y.withOutputName("declared");
+ Assertions.assertEquals(
+ new ExpressionVirtualColumn("declared", "x + y", ColumnType.FLOAT,
TestExprMacroTable.INSTANCE),
+ renamed
+ );
+ Assertions.assertEquals(X_PLUS_Y.getEquivalanceKey(),
renamed.getEquivalanceKey());
+ }
+
@Test
public void testNowCacheKeyIsNotStableAcrossInstances()
{
diff --git
a/processing/src/test/java/org/apache/druid/segment/virtual/NestedFieldVirtualColumnTest.java
b/processing/src/test/java/org/apache/druid/segment/virtual/NestedFieldVirtualColumnTest.java
index f894d2e552c..7c7b4bcf30c 100644
---
a/processing/src/test/java/org/apache/druid/segment/virtual/NestedFieldVirtualColumnTest.java
+++
b/processing/src/test/java/org/apache/druid/segment/virtual/NestedFieldVirtualColumnTest.java
@@ -98,6 +98,20 @@ public class NestedFieldVirtualColumnTest
Assertions.assertNotEquals(v1.getEquivalanceKey(), v3.getEquivalanceKey());
}
+ @Test
+ public void testWithOutputName()
+ {
+ NestedFieldVirtualColumn original = new NestedFieldVirtualColumn("nested",
"$.x.y.z", "v0", ColumnType.LONG);
+ Assertions.assertTrue(original.supportsOutputNameRewrite());
+ NestedFieldVirtualColumn renamed = original.withOutputName("declared");
+ Assertions.assertEquals(
+ new NestedFieldVirtualColumn("nested", "$.x.y.z", "declared",
ColumnType.LONG),
+ renamed
+ );
+ // Everything except the output name is identical, which the
name-independent equivalence key certifies.
+ Assertions.assertEquals(original.getEquivalanceKey(),
renamed.getEquivalanceKey());
+ }
+
@Test
public void testEqualsAndHashcode()
{
diff --git
a/processing/src/test/java/org/apache/druid/segment/virtual/NestedMergeVirtualColumnTest.java
b/processing/src/test/java/org/apache/druid/segment/virtual/NestedMergeVirtualColumnTest.java
index cdc8e334d0c..ef8c2e5a003 100644
---
a/processing/src/test/java/org/apache/druid/segment/virtual/NestedMergeVirtualColumnTest.java
+++
b/processing/src/test/java/org/apache/druid/segment/virtual/NestedMergeVirtualColumnTest.java
@@ -71,6 +71,23 @@ public class NestedMergeVirtualColumnTest
Assertions.assertEquals(Arrays.asList("col1", "col2", "col3"),
column.getColumns());
}
+ @Test
+ public void testWithOutputName()
+ {
+ NestedMergeVirtualColumn original = new NestedMergeVirtualColumn(
+ "v0",
+ Arrays.asList("col1", "col2"),
+ TestExprMacroTable.INSTANCE
+ );
+ Assertions.assertTrue(original.supportsOutputNameRewrite());
+ NestedMergeVirtualColumn renamed = original.withOutputName("declared");
+ Assertions.assertEquals(
+ new NestedMergeVirtualColumn("declared", Arrays.asList("col1",
"col2"), TestExprMacroTable.INSTANCE),
+ renamed
+ );
+ Assertions.assertEquals(original.getEquivalanceKey(),
renamed.getEquivalanceKey());
+ }
+
@Test
public void testEquivalence()
{
diff --git
a/processing/src/test/java/org/apache/druid/segment/virtual/NestedObjectVirtualColumnTest.java
b/processing/src/test/java/org/apache/druid/segment/virtual/NestedObjectVirtualColumnTest.java
index 1ac48d02fe0..678cbbd0b8f 100644
---
a/processing/src/test/java/org/apache/druid/segment/virtual/NestedObjectVirtualColumnTest.java
+++
b/processing/src/test/java/org/apache/druid/segment/virtual/NestedObjectVirtualColumnTest.java
@@ -52,6 +52,22 @@ public class NestedObjectVirtualColumnTest
Assertions.assertEquals(there, andBackAgain);
}
+ @Test
+ public void testWithOutputName()
+ {
+ Map<String, NestedObjectVirtualColumn.TypedExpression> keyExprMap =
ImmutableMap.of(
+ "computed", new NestedObjectVirtualColumn.TypedExpression("col1 +
col2", ColumnType.LONG)
+ );
+ NestedObjectVirtualColumn original = new NestedObjectVirtualColumn("v0",
keyExprMap, TestExprMacroTable.INSTANCE);
+ Assertions.assertTrue(original.supportsOutputNameRewrite());
+ NestedObjectVirtualColumn renamed = original.withOutputName("declared");
+ Assertions.assertEquals(
+ new NestedObjectVirtualColumn("declared", keyExprMap,
TestExprMacroTable.INSTANCE),
+ renamed
+ );
+ Assertions.assertEquals(original.getEquivalanceKey(),
renamed.getEquivalanceKey());
+ }
+
@Test
public void testGetKeyExprMap()
{
diff --git
a/server/src/main/java/org/apache/druid/catalog/model/BaseTableColumns.java
b/server/src/main/java/org/apache/druid/catalog/model/BaseTableColumns.java
new file mode 100644
index 00000000000..9161feec07c
--- /dev/null
+++ b/server/src/main/java/org/apache/druid/catalog/model/BaseTableColumns.java
@@ -0,0 +1,166 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.catalog.model;
+
+import org.apache.druid.data.input.impl.DimensionSchema;
+import org.apache.druid.error.InvalidInput;
+import org.apache.druid.segment.AutoTypeColumnSchema;
+import org.apache.druid.segment.DimensionHandlerUtils;
+import org.apache.druid.segment.column.ColumnHolder;
+import org.apache.druid.segment.column.ColumnType;
+import org.apache.druid.segment.column.ValueType;
+
+import javax.annotation.Nullable;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Shared logic for {@link DatasourceBaseTableMetadata} implementations to
derive the physical column list of a
+ * base-table spec from the declared catalog columns, with optional per-column
{@link DimensionSchema}
+ * customizations. Layout-specific rules (such as which columns a layout
forbids customizing) stay with the
+ * implementations.
+ */
+final class BaseTableColumns
+{
+ /**
+ * Indexes per-column schema customizations by column name, rejecting nulls
and duplicates.
+ */
+ static Map<String, DimensionSchema> indexColumnSchemas(List<DimensionSchema>
columnSchemas)
+ {
+ final Map<String, DimensionSchema> customSchemas = new HashMap<>();
+ for (DimensionSchema schema : columnSchemas) {
+ if (schema == null) {
+ throw InvalidInput.exception("columnSchemas must not contain null
entries");
+ }
+ if (customSchemas.put(schema.getName(), schema) != null) {
+ throw InvalidInput.exception("columnSchemas contains duplicate entries
for column [%s]", schema.getName());
+ }
+ }
+ return customSchemas;
+ }
+
+ /**
+ * Derives the {@link DimensionSchema} used during segment creation for one
declared catalog column: the
+ * customization when one is supplied (validated against the declared type),
else the default schema for the
+ * declared type.
+ */
+ static DimensionSchema toDimensionSchema(ColumnSpec column, @Nullable
DimensionSchema customSchema, String typeName)
+ {
+ ColumnType druidType = Columns.druidType(column);
+ if (druidType == null) {
+ // A column declared without a type defaults to STRING (mirroring
Columns.convertSignature), but a declared
+ // type that does not parse must be rejected rather than silently
defaulted: the declared type is the physical
+ // segment schema here.
+ if (column.dataType() != null) {
+ throw InvalidInput.exception(
+ "column [%s] has an unrecognized type [%s]; declare a SQL type
(such as [%s]) or a Druid type string"
+ + " (such as [%s] or [%s])",
+ column.name(),
+ column.dataType(),
+ Columns.SQL_BIGINT,
+ ColumnType.LONG_ARRAY.asTypeString(),
+ ColumnType.NESTED_DATA.asTypeString()
+ );
+ }
+ druidType = ColumnType.STRING;
+ }
+ if (customSchema != null) {
+ validateColumnSchemaCustomization(column, customSchema, druidType);
+ return customSchema;
+ }
+ if (druidType.isPrimitive() || druidType.isPrimitiveArray()) {
+ // The declared type is retained in the ingestion schema (primitive
arrays are cast, rather than left to an
+ // untyped auto column whose type is inferred from the ingested values;
note that the auto schema stores
+ // FLOAT ARRAY as DOUBLE ARRAY).
+ return DimensionSchema.getDefaultSchemaForBuiltInType(column.name(),
druidType);
+ }
+ if (druidType.is(ValueType.COMPLEX)) {
+ return DimensionHandlerUtils.getComplexDimensionSchema(column.name(),
druidType);
+ }
+ throw InvalidInput.exception(
+ "column [%s] has unsupported type [%s] for a [%s] base table",
+ column.name(),
+ druidType,
+ typeName
+ );
+ }
+
+ private static void validateColumnSchemaCustomization(
+ ColumnSpec column,
+ DimensionSchema customSchema,
+ ColumnType declaredType
+ )
+ {
+ if (ColumnHolder.TIME_COLUMN_NAME.equals(column.name())) {
+ throw InvalidInput.exception(
+ "columnSchemas cannot customize [%s]: the time column is always
stored as a long",
+ ColumnHolder.TIME_COLUMN_NAME
+ );
+ }
+ // The schema's type must match the declared logical type, so the physical
schema cannot silently contradict the
+ // SQL schema that INSERT/REPLACE queries are validated and coerced
against.
+ ColumnType expectedType = declaredType;
+ if (customSchema instanceof AutoTypeColumnSchema) {
+ // An uncast auto column stores values as they are ingested (inferring
the physical type) rather than coercing
+ // them to the declared type; only a column declared COMPLEX<json> may
store arbitrary shapes.
+ if (((AutoTypeColumnSchema) customSchema).getCastToType() == null
+ && !ColumnType.NESTED_DATA.equals(declaredType)) {
+ throw InvalidInput.exception(
+ "columnSchemas entry [%s] is an auto column schema without a
castToType; an uncast auto column stores"
+ + " values as they are ingested rather than coercing them to the
column's declared type [%s], set"
+ + " castToType to match the declared type",
+ column.name(),
+ declaredType
+ );
+ }
+ // The auto schema stores FLOAT as DOUBLE.
+ expectedType = autoColumnType(declaredType);
+ }
+ if (!expectedType.equals(customSchema.getColumnType())) {
+ throw InvalidInput.exception(
+ "columnSchemas entry [%s] of type [%s] does not match the column's
declared type [%s]; column schemas"
+ + " customize the physical representation of a declared column, not
its type",
+ column.name(),
+ customSchema.getColumnType(),
+ declaredType
+ );
+ }
+ }
+
+ /**
+ * The type the auto schema stores for a declared type: {@link
AutoTypeColumnSchema} coerces FLOAT to DOUBLE (the
+ * default derivation for declared FLOAT ARRAY columns relies on the same
coercion).
+ */
+ private static ColumnType autoColumnType(ColumnType declaredType)
+ {
+ if (ColumnType.FLOAT.equals(declaredType)) {
+ return ColumnType.DOUBLE;
+ }
+ if (ColumnType.FLOAT_ARRAY.equals(declaredType)) {
+ return ColumnType.DOUBLE_ARRAY;
+ }
+ return declaredType;
+ }
+
+ private BaseTableColumns()
+ {
+ }
+}
diff --git
a/server/src/main/java/org/apache/druid/catalog/model/ClusteredValueGroupsBaseTableMetadata.java
b/server/src/main/java/org/apache/druid/catalog/model/ClusteredValueGroupsBaseTableMetadata.java
index dd0dca87794..40b4bc99bb2 100644
---
a/server/src/main/java/org/apache/druid/catalog/model/ClusteredValueGroupsBaseTableMetadata.java
+++
b/server/src/main/java/org/apache/druid/catalog/model/ClusteredValueGroupsBaseTableMetadata.java
@@ -26,18 +26,12 @@ import com.fasterxml.jackson.annotation.JsonTypeName;
import
org.apache.druid.data.input.impl.ClusteredValueGroupsBaseTableProjectionSpec;
import org.apache.druid.data.input.impl.DimensionSchema;
import org.apache.druid.error.InvalidInput;
-import org.apache.druid.segment.AutoTypeColumnSchema;
-import org.apache.druid.segment.DimensionHandlerUtils;
import org.apache.druid.segment.VirtualColumns;
-import org.apache.druid.segment.column.ColumnHolder;
-import org.apache.druid.segment.column.ColumnType;
-import org.apache.druid.segment.column.ValueType;
import org.apache.druid.utils.CollectionUtils;
import javax.annotation.Nullable;
import java.util.ArrayList;
import java.util.Collections;
-import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
@@ -132,12 +126,21 @@ public class ClusteredValueGroupsBaseTableMetadata
implements DatasourceBaseTabl
TYPE_NAME
);
}
- final Map<String, DimensionSchema> customSchemas = indexColumnSchemas();
+ final Map<String, DimensionSchema> customSchemas =
BaseTableColumns.indexColumnSchemas(columnSchemas);
+ for (String customized : customSchemas.keySet()) {
+ if (clusteringColumns.contains(customized)) {
+ throw InvalidInput.exception(
+ "columnSchemas cannot customize clustering column [%s]: the
physical representation of clustering columns"
+ + " is fixed by the clustered segment format",
+ customized
+ );
+ }
+ }
final Set<String> declaredNames = new HashSet<>();
final List<DimensionSchema> specColumns = new ArrayList<>(columns.size());
for (ColumnSpec column : columns) {
declaredNames.add(column.name());
- specColumns.add(toDimensionSchema(column,
customSchemas.get(column.name())));
+ specColumns.add(BaseTableColumns.toDimensionSchema(column,
customSchemas.get(column.name()), TYPE_NAME));
}
for (String clusteringColumn : clusteringColumns) {
if (!declaredNames.contains(clusteringColumn)) {
@@ -166,124 +169,6 @@ public class ClusteredValueGroupsBaseTableMetadata
implements DatasourceBaseTabl
.build();
}
- private Map<String, DimensionSchema> indexColumnSchemas()
- {
- final Map<String, DimensionSchema> customSchemas = new HashMap<>();
- for (DimensionSchema schema : columnSchemas) {
- if (schema == null) {
- throw InvalidInput.exception("columnSchemas must not contain null
entries");
- }
- if (customSchemas.put(schema.getName(), schema) != null) {
- throw InvalidInput.exception("columnSchemas contains duplicate entries
for column [%s]", schema.getName());
- }
- }
- return customSchemas;
- }
-
- private DimensionSchema toDimensionSchema(ColumnSpec column, @Nullable
DimensionSchema customSchema)
- {
- ColumnType druidType = Columns.druidType(column);
- if (druidType == null) {
- // A column declared without a type defaults to STRING (mirroring
Columns.convertSignature), but a declared
- // type that does not parse must be rejected rather than silently
defaulted: the declared type is the physical
- // segment schema here.
- if (column.dataType() != null) {
- throw InvalidInput.exception(
- "column [%s] has an unrecognized type [%s]; declare a SQL type
(such as [%s]) or a Druid type string"
- + " (such as [%s] or [%s])",
- column.name(),
- column.dataType(),
- Columns.SQL_BIGINT,
- ColumnType.LONG_ARRAY.asTypeString(),
- ColumnType.NESTED_DATA.asTypeString()
- );
- }
- druidType = ColumnType.STRING;
- }
- if (customSchema != null) {
- validateColumnSchemaCustomization(column, customSchema, druidType);
- return customSchema;
- }
- if (druidType.isPrimitive() || druidType.isPrimitiveArray()) {
- // The declared type is retained in the ingestion schema (primitive
arrays are cast, rather than left to an
- // untyped auto column whose type is inferred from the ingested values;
note that the auto schema stores
- // FLOAT ARRAY as DOUBLE ARRAY).
- return DimensionSchema.getDefaultSchemaForBuiltInType(column.name(),
druidType);
- }
- if (druidType.is(ValueType.COMPLEX)) {
- return DimensionHandlerUtils.getComplexDimensionSchema(column.name(),
druidType);
- }
- throw InvalidInput.exception(
- "column [%s] has unsupported type [%s] for a clustered base table",
- column.name(),
- druidType
- );
- }
-
- private void validateColumnSchemaCustomization(
- ColumnSpec column,
- DimensionSchema customSchema,
- ColumnType declaredType
- )
- {
- if (ColumnHolder.TIME_COLUMN_NAME.equals(column.name())) {
- throw InvalidInput.exception(
- "columnSchemas cannot customize [%s]: the time column is always
stored as a long",
- ColumnHolder.TIME_COLUMN_NAME
- );
- }
- if (clusteringColumns.contains(column.name())) {
- throw InvalidInput.exception(
- "columnSchemas cannot customize clustering column [%s]: the physical
representation of clustering columns"
- + " is fixed by the clustered segment format",
- column.name()
- );
- }
- // The schema's type must match the declared logical type, so the physical
schema cannot silently contradict the
- // SQL schema that INSERT/REPLACE queries are validated and coerced
against.
- ColumnType expectedType = declaredType;
- if (customSchema instanceof AutoTypeColumnSchema) {
- // An uncast auto column stores values as they are ingested (inferring
the physical type) rather than coercing
- // them to the declared type; only a column declared COMPLEX<json> may
store arbitrary shapes.
- if (((AutoTypeColumnSchema) customSchema).getCastToType() == null
- && !ColumnType.NESTED_DATA.equals(declaredType)) {
- throw InvalidInput.exception(
- "columnSchemas entry [%s] is an auto column schema without a
castToType; an uncast auto column stores"
- + " values as they are ingested rather than coercing them to the
column's declared type [%s], set"
- + " castToType to match the declared type",
- column.name(),
- declaredType
- );
- }
- // The auto schema stores FLOAT as DOUBLE.
- expectedType = autoColumnType(declaredType);
- }
- if (!expectedType.equals(customSchema.getColumnType())) {
- throw InvalidInput.exception(
- "columnSchemas entry [%s] of type [%s] does not match the column's
declared type [%s]; column schemas"
- + " customize the physical representation of a declared column, not
its type",
- column.name(),
- customSchema.getColumnType(),
- declaredType
- );
- }
- }
-
- /**
- * The type the auto schema stores for a declared type: {@link
AutoTypeColumnSchema} coerces FLOAT to DOUBLE (the
- * default derivation for declared FLOAT ARRAY columns relies on the same
coercion).
- */
- private static ColumnType autoColumnType(ColumnType declaredType)
- {
- if (ColumnType.FLOAT.equals(declaredType)) {
- return ColumnType.DOUBLE;
- }
- if (ColumnType.FLOAT_ARRAY.equals(declaredType)) {
- return ColumnType.DOUBLE_ARRAY;
- }
- return declaredType;
- }
-
@Override
public boolean equals(Object o)
{
diff --git
a/server/src/main/java/org/apache/druid/catalog/model/DatasourceBaseTableMetadata.java
b/server/src/main/java/org/apache/druid/catalog/model/DatasourceBaseTableMetadata.java
index b73cfd7f4b4..17b2de20020 100644
---
a/server/src/main/java/org/apache/druid/catalog/model/DatasourceBaseTableMetadata.java
+++
b/server/src/main/java/org/apache/druid/catalog/model/DatasourceBaseTableMetadata.java
@@ -36,6 +36,7 @@ import java.util.List;
*/
@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include =
JsonTypeInfo.As.EXISTING_PROPERTY, property = "type")
@JsonSubTypes(value = {
+ @JsonSubTypes.Type(name = TableBaseTableMetadata.TYPE_NAME, value =
TableBaseTableMetadata.class),
@JsonSubTypes.Type(
name = ClusteredValueGroupsBaseTableMetadata.TYPE_NAME,
value = ClusteredValueGroupsBaseTableMetadata.class
diff --git
a/server/src/main/java/org/apache/druid/catalog/model/TableBaseTableMetadata.java
b/server/src/main/java/org/apache/druid/catalog/model/TableBaseTableMetadata.java
new file mode 100644
index 00000000000..ae0048d2382
--- /dev/null
+++
b/server/src/main/java/org/apache/druid/catalog/model/TableBaseTableMetadata.java
@@ -0,0 +1,164 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.catalog.model;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonTypeName;
+import org.apache.druid.data.input.impl.DimensionSchema;
+import org.apache.druid.data.input.impl.TableProjectionSpec;
+import org.apache.druid.error.InvalidInput;
+import org.apache.druid.segment.VirtualColumns;
+import org.apache.druid.utils.CollectionUtils;
+
+import javax.annotation.Nullable;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+
+/**
+ * Catalog layout metadata for plain {@link TableProjectionSpec} base tables.
The layout has no shape of its own beyond
+ * the catalog column list (declared column order is the physical segment
storage and sort order), so this metadata
+ * carries only the optional {@link #virtualColumns},which the plain spec
restricts to the query-granularity carrier,
+ * and optional per-column {@link #columnSchemas} customizations; {@link
#createSpec(List)} combines them with the
+ * declared columns into the physical spec used to generate segments.
+ * <p>
+ * The optional {@link #columnSchemas} do NOT define columns, the catalog
column list remains the logical schema,
+ * they are per-column customizations of the exact {@link DimensionSchema}
used to create segments for a declared
+ * column, replacing the default schema derived from the declared column type.
+ */
+@JsonTypeName(TableBaseTableMetadata.TYPE_NAME)
+public class TableBaseTableMetadata implements DatasourceBaseTableMetadata
+{
+ public static final String TYPE_NAME = TableProjectionSpec.TYPE_NAME;
+
+ private final VirtualColumns virtualColumns;
+ private final List<DimensionSchema> columnSchemas;
+
+ @JsonCreator
+ public TableBaseTableMetadata(
+ @JsonProperty("virtualColumns") @Nullable VirtualColumns virtualColumns,
+ @JsonProperty("columnSchemas") @Nullable List<DimensionSchema>
columnSchemas
+ )
+ {
+ this.virtualColumns = virtualColumns == null ? VirtualColumns.EMPTY :
virtualColumns;
+ this.columnSchemas = columnSchemas == null ? Collections.emptyList() :
columnSchemas;
+ }
+
+ @Override
+ @JsonProperty("type")
+ public String getType()
+ {
+ return TYPE_NAME;
+ }
+
+ @Override
+ @JsonProperty("virtualColumns")
+ @JsonInclude(JsonInclude.Include.NON_DEFAULT)
+ public VirtualColumns getVirtualColumns()
+ {
+ return virtualColumns;
+ }
+
+ /**
+ * Per-column customizations of the {@link DimensionSchema} used during
segment creation, keyed by
+ * {@link DimensionSchema#getName()}; empty when every declared column uses
the schema derived from its declared
+ * type. These do not define columns: every entry must customize a declared
column.
+ */
+ @JsonProperty("columnSchemas")
+ @JsonInclude(JsonInclude.Include.NON_EMPTY)
+ public List<DimensionSchema> getColumnSchemas()
+ {
+ return columnSchemas;
+ }
+
+ /**
+ * Creates the physical spec from the declared catalog columns, used
verbatim: the declared column order is the
+ * physical segment order and sort order. All layout rules — the explicit
{@code __time} position, and the
+ * restriction of {@link #virtualColumns} to the query-granularity carrier —
are enforced by the spec itself.
+ * <p>
+ * A column with an entry in {@link #columnSchemas} uses that {@link
DimensionSchema} verbatim in place of the
+ * default derived from its declared type. Customizations may only target
declared columns ({@code __time} is always
+ * a long), and the schema's type must match the declared logical type so
the physical schema cannot silently
+ * contradict the SQL schema.
+ */
+ @Override
+ public TableProjectionSpec createSpec(List<ColumnSpec> columns)
+ {
+ if (CollectionUtils.isNullOrEmpty(columns)) {
+ throw InvalidInput.exception(
+ "Cannot define a [%s] base table without declared columns; the
catalog column list defines the table schema",
+ TYPE_NAME
+ );
+ }
+ final Map<String, DimensionSchema> customSchemas =
BaseTableColumns.indexColumnSchemas(columnSchemas);
+ final Set<String> declaredNames = new HashSet<>();
+ final List<DimensionSchema> specColumns = new ArrayList<>(columns.size());
+ for (ColumnSpec column : columns) {
+ declaredNames.add(column.name());
+ specColumns.add(BaseTableColumns.toDimensionSchema(column,
customSchemas.get(column.name()), TYPE_NAME));
+ }
+ for (String customized : customSchemas.keySet()) {
+ if (!declaredNames.contains(customized)) {
+ throw InvalidInput.exception(
+ "columnSchemas entry [%s] does not customize a declared column;
column schemas do not define columns,"
+ + " declare [%s] in the table's column list",
+ customized,
+ customized
+ );
+ }
+ }
+ return TableProjectionSpec.builder()
+ .virtualColumns(virtualColumns)
+ .columns(specColumns)
+ .build();
+ }
+
+ @Override
+ public boolean equals(Object o)
+ {
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ TableBaseTableMetadata that = (TableBaseTableMetadata) o;
+ return Objects.equals(virtualColumns, that.virtualColumns)
+ && Objects.equals(columnSchemas, that.columnSchemas);
+ }
+
+ @Override
+ public int hashCode()
+ {
+ return Objects.hash(virtualColumns, columnSchemas);
+ }
+
+ @Override
+ public String toString()
+ {
+ return "TableBaseTableMetadata{" +
+ "virtualColumns=" + virtualColumns +
+ ", columnSchemas=" + columnSchemas +
+ '}';
+ }
+}
diff --git
a/server/src/main/java/org/apache/druid/segment/indexing/DataSchema.java
b/server/src/main/java/org/apache/druid/segment/indexing/DataSchema.java
index 741fadf9406..51051337a3a 100644
--- a/server/src/main/java/org/apache/druid/segment/indexing/DataSchema.java
+++ b/server/src/main/java/org/apache/druid/segment/indexing/DataSchema.java
@@ -247,10 +247,7 @@ public class DataSchema
private static Granularity queryGranularityFromSpec(BaseTableProjectionSpec
spec)
{
final VirtualColumn vc =
spec.getVirtualColumns().getVirtualColumn(Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME);
- if (vc == null) {
- return Granularities.NONE;
- }
- final Granularity granularity = Granularities.fromVirtualColumn(vc);
+ final Granularity granularity = Granularities.fromTimeVirtualColumn(vc);
return granularity == null ? Granularities.NONE : granularity;
}
diff --git
a/server/src/test/java/org/apache/druid/catalog/model/TableBaseTableMetadataTest.java
b/server/src/test/java/org/apache/druid/catalog/model/TableBaseTableMetadataTest.java
new file mode 100644
index 00000000000..fd05651b1aa
--- /dev/null
+++
b/server/src/test/java/org/apache/druid/catalog/model/TableBaseTableMetadataTest.java
@@ -0,0 +1,190 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.catalog.model;
+
+import com.fasterxml.jackson.databind.InjectableValues;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.druid.data.input.impl.DimensionSchema;
+import org.apache.druid.data.input.impl.StringDimensionSchema;
+import org.apache.druid.data.input.impl.TableProjectionSpec;
+import org.apache.druid.error.DruidException;
+import org.apache.druid.guice.BuiltInTypesModule;
+import org.apache.druid.jackson.DefaultObjectMapper;
+import org.apache.druid.java.util.common.granularity.Granularities;
+import org.apache.druid.math.expr.ExprMacroTable;
+import org.apache.druid.query.OrderBy;
+import org.apache.druid.segment.DefaultColumnFormatConfig;
+import org.apache.druid.segment.VirtualColumns;
+import org.apache.druid.testing.InitializedNullHandlingTest;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.stream.Collectors;
+
+/**
+ * The column-derivation rules shared with the clustered layout (type
retention, complex handler resolution,
+ * columnSchemas customization validation) are covered exhaustively in {@link
ClusteredValueGroupsBaseTableMetadataTest}
+ * and route through the same {@code BaseTableColumns} helper; this covers
what is specific to the plain layout.
+ */
+public class TableBaseTableMetadataTest extends InitializedNullHandlingTest
+{
+ static {
+ BuiltInTypesModule.registerHandlersAndSerde();
+ }
+
+ // The granularity macro table (not nil) so the carrier's timestamp_floor
expression can be parsed back.
+ private final ObjectMapper mapper = new
DefaultObjectMapper().setInjectableValues(
+ new InjectableValues.Std()
+ .addValue(ExprMacroTable.class, ExprMacroTable.granularity())
+ .addValue(DefaultColumnFormatConfig.class, new
DefaultColumnFormatConfig(null, null, null, null))
+ );
+
+ private static final List<ColumnSpec> COLUMNS = Arrays.asList(
+ new ColumnSpec("tenant", Columns.SQL_VARCHAR, null),
+ new ColumnSpec(Columns.TIME_COLUMN, Columns.SQL_TIMESTAMP, null),
+ new ColumnSpec("delta", Columns.SQL_BIGINT, null)
+ );
+
+ private static VirtualColumns hourCarrier()
+ {
+ return VirtualColumns.create(
+ Granularities.toVirtualColumn(Granularities.HOUR,
Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME)
+ );
+ }
+
+ @Test
+ public void testSerde() throws Exception
+ {
+ final DatasourceBaseTableMetadata metadata = new
TableBaseTableMetadata(hourCarrier(), null);
+ final String json = mapper.writeValueAsString(metadata);
+ Assertions.assertTrue(json.contains("\"type\":\"table\""), json);
+ final DatasourceBaseTableMetadata fromJson = mapper.readValue(json,
DatasourceBaseTableMetadata.class);
+ Assertions.assertEquals(metadata, fromJson);
+ }
+
+ @Test
+ public void testSerdeEmpty() throws Exception
+ {
+ final DatasourceBaseTableMetadata metadata = new
TableBaseTableMetadata(null, null);
+ final String json = mapper.writeValueAsString(metadata);
+ Assertions.assertFalse(json.contains("virtualColumns"));
+ Assertions.assertFalse(json.contains("columnSchemas"));
+ final DatasourceBaseTableMetadata fromJson = mapper.readValue(json,
DatasourceBaseTableMetadata.class);
+ Assertions.assertEquals(metadata, fromJson);
+ }
+
+ @Test
+ public void testCreateSpec()
+ {
+ final TableProjectionSpec spec = new TableBaseTableMetadata(hourCarrier(),
null).createSpec(COLUMNS);
+
+ // Declared catalog order is both storage and sort order, and the carrier
rides through as the query granularity.
+ Assertions.assertEquals(
+ List.of("tenant", Columns.TIME_COLUMN, "delta"),
+
spec.getColumns().stream().map(DimensionSchema::getName).collect(Collectors.toList())
+ );
+ Assertions.assertEquals(
+ List.of(
+ OrderBy.ascending("tenant"),
+ OrderBy.ascending(Columns.TIME_COLUMN),
+ OrderBy.ascending("delta")
+ ),
+ spec.getOrdering()
+ );
+ Assertions.assertEquals(Granularities.HOUR, spec.getQueryGranularity());
+ Assertions.assertEquals(0, spec.getMetrics().length);
+ }
+
+ @Test
+ public void testCreateSpecWithColumnSchemas()
+ {
+ final TableProjectionSpec spec = new TableBaseTableMetadata(
+ null,
+ Collections.singletonList(
+ new StringDimensionSchema("tenant",
DimensionSchema.MultiValueHandling.ARRAY, false)
+ )
+ ).createSpec(COLUMNS);
+
+ Assertions.assertEquals(
+ new StringDimensionSchema("tenant",
DimensionSchema.MultiValueHandling.ARRAY, false),
+ spec.getColumns().get(0)
+ );
+ }
+
+ @Test
+ public void testCreateSpecColumnSchemaForUndeclaredColumnFails()
+ {
+ final DruidException e = Assertions.assertThrows(
+ DruidException.class,
+ () -> new TableBaseTableMetadata(
+ null,
+ Collections.singletonList(new StringDimensionSchema("nope"))
+ ).createSpec(COLUMNS)
+ );
+ Assertions.assertTrue(e.getMessage().contains("columnSchemas entry [nope]
does not customize a declared column"));
+ }
+
+ /**
+ * The plain spec accepts only the granularity carrier, so any other virtual
column in the metadata surfaces as the
+ * spec's rejection when the layout is validated.
+ */
+ @Test
+ public void testCreateSpecNonCarrierVirtualColumnFails()
+ {
+ final DruidException e = Assertions.assertThrows(
+ DruidException.class,
+ () -> new TableBaseTableMetadata(
+ VirtualColumns.create(
+ Granularities.toVirtualColumn(Granularities.HOUR,
"not_the_carrier")
+ ),
+ null
+ ).createSpec(COLUMNS)
+ );
+ Assertions.assertTrue(e.getMessage().contains("virtual column
[not_the_carrier] is not supported"), e.getMessage());
+ }
+
+ @Test
+ public void testCreateSpecMissingTimeColumnFails()
+ {
+ final DruidException e = Assertions.assertThrows(
+ DruidException.class,
+ () -> new TableBaseTableMetadata(null, null).createSpec(
+ Collections.singletonList(new ColumnSpec("tenant",
Columns.SQL_VARCHAR, null))
+ )
+ );
+ Assertions.assertTrue(e.getMessage().contains(Columns.TIME_COLUMN),
e.getMessage());
+ }
+
+ @Test
+ public void testCreateSpecNoDeclaredColumnsFails()
+ {
+ final DruidException e = Assertions.assertThrows(
+ DruidException.class,
+ () -> new TableBaseTableMetadata(null, null).createSpec(null)
+ );
+ Assertions.assertTrue(
+ e.getMessage().contains("Cannot define a [table] base table without
declared columns"),
+ e.getMessage()
+ );
+ }
+}
diff --git
a/server/src/test/java/org/apache/druid/segment/indexing/DataSchemaTest.java
b/server/src/test/java/org/apache/druid/segment/indexing/DataSchemaTest.java
index 0bf965fe5b0..7836f057d32 100644
--- a/server/src/test/java/org/apache/druid/segment/indexing/DataSchemaTest.java
+++ b/server/src/test/java/org/apache/druid/segment/indexing/DataSchemaTest.java
@@ -35,6 +35,7 @@ import
org.apache.druid.data.input.impl.ClusteredValueGroupsBaseTableProjectionS
import org.apache.druid.data.input.impl.DimensionsSpec;
import org.apache.druid.data.input.impl.LongDimensionSchema;
import org.apache.druid.data.input.impl.StringDimensionSchema;
+import org.apache.druid.data.input.impl.TableProjectionSpec;
import org.apache.druid.data.input.impl.TimestampSpec;
import org.apache.druid.error.DruidException;
import org.apache.druid.error.DruidExceptionMatcher;
@@ -927,6 +928,47 @@ class DataSchemaTest extends InitializedNullHandlingTest
Assertions.assertEquals(spec, deserialized.getBaseTable());
}
+ /**
+ * The generic baseTable-mode machinery is covered above with the clustered
spec; this covers what is specific to a
+ * plain {@link TableProjectionSpec}: it rides the same {@code baseTable}
property (JSON type {@code table}), and it
+ * lowers into the standard ingest inputs — a dimensionsSpec with the
explicit {@code __time} position and
+ * {@code forceSegmentSortByTime=false}, no metrics, and a recombined
granularity with rollup off and query
+ * granularity read from the spec's carrier virtual column.
+ */
+ @Test
+ void testBaseTableModeWithTableProjectionSpec() throws IOException
+ {
+ final TableProjectionSpec spec = TableProjectionSpec.builder()
+ .columns(
+ new StringDimensionSchema("page"),
+ new LongDimensionSchema("__time"),
+ new LongDimensionSchema("cnt")
+ )
+ .build()
+ .withQueryGranularity(Granularities.HOUR);
+ final DataSchema original = DataSchema.builder()
+ .withDataSource("datasource")
+ .withTimestamp(TIMESTAMP_SPEC)
+ .withSegmentGranularity(new
SegmentGranularitySpec(Granularities.DAY, null))
+ .withBaseTable(spec)
+ .build();
+
+ final String serialized = jsonMapper.writeValueAsString(original);
+ final JsonNode root = jsonMapper.readTree(serialized);
+ Assertions.assertEquals("table",
root.get("baseTable").get("type").asText());
+ final DataSchema deserialized = jsonMapper.readValue(serialized,
DataSchema.class);
+ Assertions.assertEquals(original, deserialized);
+ Assertions.assertEquals(spec, deserialized.getBaseTable());
+
+ Assertions.assertEquals(spec.getDimensionsSpec(),
original.getDimensionsSpec());
+
Assertions.assertFalse(original.getDimensionsSpec().isForceSegmentSortByTime());
+ Assertions.assertEquals(0, original.getAggregators().length);
+ final GranularitySpec effectiveGranularity = original.getGranularitySpec();
+ Assertions.assertEquals(Granularities.DAY,
effectiveGranularity.getSegmentGranularity());
+ Assertions.assertEquals(Granularities.HOUR,
effectiveGranularity.getQueryGranularity());
+ Assertions.assertFalse(effectiveGranularity.isRollup());
+ }
+
@Test
void testBaseTableModeLegacyAccessorsDelegateToSpec()
{
diff --git
a/sql/src/main/java/org/apache/druid/sql/calcite/planner/CatalogDdlHandler.java
b/sql/src/main/java/org/apache/druid/sql/calcite/planner/CatalogDdlHandler.java
index b4f315f0213..a59aaa730d0 100644
---
a/sql/src/main/java/org/apache/druid/sql/calcite/planner/CatalogDdlHandler.java
+++
b/sql/src/main/java/org/apache/druid/sql/calcite/planner/CatalogDdlHandler.java
@@ -30,9 +30,9 @@ import org.apache.calcite.sql.SqlLiteral;
import org.apache.calcite.sql.SqlNode;
import org.apache.calcite.sql.SqlNodeList;
import org.apache.calcite.sql.type.SqlTypeName;
-import org.apache.druid.catalog.model.ClusteredValueGroupsBaseTableMetadata;
import org.apache.druid.catalog.model.ColumnSpec;
import org.apache.druid.catalog.model.Columns;
+import org.apache.druid.catalog.model.DatasourceBaseTableMetadata;
import org.apache.druid.catalog.model.DatasourceProjectionMetadata;
import org.apache.druid.catalog.model.TableId;
import org.apache.druid.catalog.model.TableMetadata;
@@ -275,7 +275,7 @@ public abstract class CatalogDdlHandler extends
SqlStatementHandler.BaseStatemen
* Translate the reserved {@code __base} projection, which describes the
physical layout of the table rather than an
* additional aggregate, and so becomes the {@code baseTable} property
instead of one of the projections.
*/
- protected static ClusteredValueGroupsBaseTableMetadata translateBaseTable(
+ protected static DatasourceBaseTableMetadata translateBaseTable(
final SqlStatementHandler.HandlerContext handlerContext,
final String tableName,
final List<ColumnSpec> columns,
diff --git
a/sql/src/main/java/org/apache/druid/sql/calcite/planner/ProjectionSpecTranslator.java
b/sql/src/main/java/org/apache/druid/sql/calcite/planner/ProjectionSpecTranslator.java
index c401639f620..dbb48b78f82 100644
---
a/sql/src/main/java/org/apache/druid/sql/calcite/planner/ProjectionSpecTranslator.java
+++
b/sql/src/main/java/org/apache/druid/sql/calcite/planner/ProjectionSpecTranslator.java
@@ -30,13 +30,16 @@ import org.apache.calcite.sql.util.SqlBasicVisitor;
import org.apache.druid.catalog.model.ClusteredValueGroupsBaseTableMetadata;
import org.apache.druid.catalog.model.ColumnSpec;
import org.apache.druid.catalog.model.Columns;
+import org.apache.druid.catalog.model.DatasourceBaseTableMetadata;
+import org.apache.druid.catalog.model.TableBaseTableMetadata;
import org.apache.druid.data.input.impl.AggregateProjectionSpec;
import org.apache.druid.data.input.impl.DimensionSchema;
import org.apache.druid.error.DruidException;
import org.apache.druid.error.InvalidSqlInput;
import org.apache.druid.java.util.common.Intervals;
import org.apache.druid.java.util.common.StringUtils;
-import org.apache.druid.math.expr.ExprMacroTable;
+import org.apache.druid.java.util.common.granularity.Granularities;
+import org.apache.druid.java.util.common.granularity.Granularity;
import org.apache.druid.query.DataSource;
import org.apache.druid.query.Query;
import org.apache.druid.query.QueryContexts;
@@ -52,9 +55,9 @@ import org.apache.druid.query.scan.ScanQuery;
import org.apache.druid.query.timeseries.TimeseriesQuery;
import org.apache.druid.segment.VirtualColumn;
import org.apache.druid.segment.VirtualColumns;
+import org.apache.druid.segment.column.ColumnHolder;
import org.apache.druid.segment.column.ColumnType;
import org.apache.druid.segment.column.RowSignature;
-import org.apache.druid.segment.virtual.ExpressionVirtualColumn;
import org.apache.druid.server.security.AuthorizationResult;
import org.apache.druid.server.security.NoopEscalator;
import org.apache.druid.sql.calcite.rel.DruidQuery;
@@ -68,8 +71,11 @@ import javax.annotation.Nullable;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
+import java.util.Set;
/**
* Turns the SQL body of a projection definition into the {@link
AggregateProjectionSpec} the catalog stores.
@@ -139,13 +145,17 @@ public class ProjectionSpecTranslator
* than an additional aggregate.
* <p>
* The body enumerates the table's columns in the order segments store them,
so it must name every declared column,
- * in declared order. An item written as {@code <expr> AS <name>} makes that
column computed at ingest time: the
- * expression becomes a virtual column materializing the declared column,
which is why the declared type has to
- * match what the expression produces.
+ * in declared order. With {@code CLUSTERED BY} the layout is a clustered
base table, and an item written as
+ * {@code <expr> AS <name>} makes that column computed at ingest time: the
expression becomes a virtual column
+ * materializing the declared column, which is why the declared type has to
match what the expression produces.
+ * Without {@code CLUSTERED BY} the layout is a plain table, which stores
columns as they arrive; the only
+ * expression it accepts is {@code TIME_FLOOR(__time, <period>)} selected as
{@code __time}, which becomes the
+ * table's query granularity (carried as the granularity virtual column of
the spec).
*
- * @param clusteredBy the columns segments are clustered on, which must be
the leading prefix of the column list
+ * @param clusteredBy the columns segments are clustered on, which must be
the leading prefix of the column list;
+ * null selects the plain-table layout
*/
- public ClusteredValueGroupsBaseTableMetadata translateBaseTable(
+ public DatasourceBaseTableMetadata translateBaseTable(
final String tableName,
final List<ColumnSpec> columns,
final SqlSelect body,
@@ -161,11 +171,19 @@ public class ProjectionSpecTranslator
rejectSubqueries(BASE_PROJECTION_NAME, body);
final DruidQuery druidQuery = planBody(tableName, columns,
BASE_PROJECTION_NAME, body);
- final ClusteredValueGroupsBaseTableMetadata metadata = new
ClusteredValueGroupsBaseTableMetadata(
- clusteringColumns(clusteredBy),
- liftComputedColumns(columns, druidQuery),
- null
- );
+ final List<ComputedColumn> computed = liftComputedColumns(columns,
druidQuery);
+ final DatasourceBaseTableMetadata metadata;
+ if (clusteredBy == null) {
+ metadata = new
TableBaseTableMetadata(plainTableVirtualColumns(computed), null);
+ } else {
+ // liftComputedColumns already verified that the body plans to a scan.
+ final VirtualColumns planned = druidQuery.getQuery().getVirtualColumns();
+ metadata = new ClusteredValueGroupsBaseTableMetadata(
+ clusteringColumns(clusteredBy),
+ materializedVirtualColumns(computed, planned, columns),
+ null
+ );
+ }
// Derive the physical spec now. The catalog does this too when the write
lands, but doing it here attributes
// layout problems to the statement that caused them rather than to a
Coordinator round trip.
@@ -178,6 +196,98 @@ public class ProjectionSpecTranslator
return metadata;
}
+ /**
+ * A computed column of a plain (non-clustered) base table can only be the
table's query granularity.
+ */
+ private static VirtualColumns plainTableVirtualColumns(final
List<ComputedColumn> computed)
+ {
+ final List<VirtualColumn> virtualColumns = new ArrayList<>();
+ for (ComputedColumn column : computed) {
+ if (!ColumnHolder.TIME_COLUMN_NAME.equals(column.declaredName)) {
+ throw invalid(
+ BASE_PROJECTION_NAME,
+ StringUtils.format(
+ "its column [%s] is computed by an expression. Computed
columns are not supported; compute the column"
+ + " at ingestion time instead",
+ column.declaredName
+ )
+ );
+ }
+ final Granularity granularity =
Granularities.fromTimeVirtualColumn(column.virtualColumn);
+ if (granularity == null) {
+ throw invalid(
+ BASE_PROJECTION_NAME,
+ StringUtils.format(
+ "it computes [%s] with an expression that is not a
granularity. A base table computes [%s] only as"
+ + " TIME_FLOOR(%s, <period>), which declares the table's query
granularity",
+ ColumnHolder.TIME_COLUMN_NAME,
+ ColumnHolder.TIME_COLUMN_NAME,
+ ColumnHolder.TIME_COLUMN_NAME
+ )
+ );
+ }
+ virtualColumns.add(Granularities.toVirtualColumn(granularity,
Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME));
+ }
+ return VirtualColumns.create(virtualColumns);
+ }
+
+ /**
+ * The clustered write path materializes computed columns: each planned
virtual column is stored under the name of
+ * the declared column it fills, together with its dependency closure. The
planner composes specialized virtual
+ * columns by reference so the intermediaries a materialized column reads
are carried into the spec under those
+ * synthetic names, as unstored inputs of the columns they feed. Only the
materialized roots are renamed.
+ */
+ private static VirtualColumns materializedVirtualColumns(
+ final List<ComputedColumn> computed,
+ final VirtualColumns planned,
+ final List<ColumnSpec> columns
+ )
+ {
+ final Set<String> declared = new HashSet<>();
+ for (ColumnSpec column : columns) {
+ declared.add(column.name());
+ }
+ final List<VirtualColumn> materialized = new ArrayList<>(computed.size());
+ final Map<String, VirtualColumn> intermediaries = new LinkedHashMap<>();
+ for (ComputedColumn column : computed) {
+ if (!column.virtualColumn.supportsOutputNameRewrite()) {
+ throw invalid(
+ BASE_PROJECTION_NAME,
+ "column [" + column.declaredName + "] is computed by an expression
the base table cannot store"
+ );
+ }
+
materialized.add(column.virtualColumn.withOutputName(column.declaredName));
+ collectDependencies(column.virtualColumn, planned, declared,
intermediaries);
+ }
+ materialized.addAll(intermediaries.values());
+ return VirtualColumns.create(materialized);
+ }
+
+ /**
+ * Collects the planned virtual columns that {@code root} (transitively)
reads by synthetic name. A required column
+ * that is declared is a stored column reference and needs nothing; one that
names a planned virtual column is an
+ * intermediary to carry along; anything else is left for the spec's own
validation to report.
+ */
+ private static void collectDependencies(
+ final VirtualColumn root,
+ final VirtualColumns planned,
+ final Set<String> declared,
+ final Map<String, VirtualColumn> intermediaries
+ )
+ {
+ for (String input : root.requiredColumns()) {
+ if (declared.contains(input) || intermediaries.containsKey(input)) {
+ continue;
+ }
+ final VirtualColumn dependency = planned.getVirtualColumn(input);
+ if (dependency == null) {
+ continue;
+ }
+ intermediaries.put(input, dependency);
+ collectDependencies(dependency, planned, declared, intermediaries);
+ }
+ }
+
private static List<String> clusteringColumns(@Nullable final SqlNodeList
clusteredBy)
{
if (clusteredBy == null) {
@@ -198,12 +308,30 @@ public class ProjectionSpecTranslator
}
/**
- * Pair the planned output with the declared columns and lift the virtual
columns behind the computed ones.
+ * A body item written as {@code <expr> AS <name>}: the declared column it
fills, paired with the planned virtual
+ * column that computes it. The virtual column is exactly what the planner
would generate for the same expression in
+ * a query — including specializations such as the nested-field virtual
column — so a layout can interpret it before
+ * deciding how (or whether) to store it, and what it stores matches what
queries plan.
+ */
+ private static final class ComputedColumn
+ {
+ private final String declaredName;
+ private final VirtualColumn virtualColumn;
+
+ private ComputedColumn(String declaredName, VirtualColumn virtualColumn)
+ {
+ this.declaredName = declaredName;
+ this.virtualColumn = virtualColumn;
+ }
+ }
+
+ /**
+ * Pair the planned output with the declared columns and lift the
expressions behind the computed ones.
* <p>
- * The planner names its virtual columns {@code v0}, {@code v1}, ...; each
is renamed to the declared column it
+ * The planner names its virtual columns {@code v0}, {@code v1}, ...; each
is paired with the declared column it
* fills, which is what makes it a materialized column rather than an
anonymous intermediate.
*/
- private static VirtualColumns liftComputedColumns(
+ private static List<ComputedColumn> liftComputedColumns(
final List<ColumnSpec> columns,
final DruidQuery druidQuery
)
@@ -230,8 +358,8 @@ public class ProjectionSpecTranslator
);
}
- final VirtualColumns planned = ((ScanQuery) query).getVirtualColumns();
- final List<VirtualColumn> materialized = new ArrayList<>();
+ final VirtualColumns planned = query.getVirtualColumns();
+ final List<ComputedColumn> computed = new ArrayList<>();
for (int i = 0; i < columns.size(); i++) {
final String declared = columns.get(i).name();
if (!declared.equals(outputNames.get(i))) {
@@ -265,23 +393,9 @@ public class ProjectionSpecTranslator
// A plain reference: the column is ingested as it arrives.
continue;
}
- if (!(virtualColumn instanceof ExpressionVirtualColumn)) {
- throw invalid(
- BASE_PROJECTION_NAME,
- "column [" + declared + "] is computed by an expression the base
table cannot store"
- );
- }
- final ExpressionVirtualColumn expression = (ExpressionVirtualColumn)
virtualColumn;
- materialized.add(
- new ExpressionVirtualColumn(
- declared,
- expression.getExpression(),
- expression.getOutputType(),
- ExprMacroTable.nil()
- )
- );
+ computed.add(new ComputedColumn(declared, virtualColumn));
}
- return VirtualColumns.create(materialized);
+ return computed;
}
/**
diff --git
a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteCatalogDdlTest.java
b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteCatalogDdlTest.java
index d57b1c34092..f6d65b5f4d7 100644
--- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteCatalogDdlTest.java
+++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteCatalogDdlTest.java
@@ -31,6 +31,7 @@ import
org.apache.druid.catalog.model.ClusteredValueGroupsBaseTableMetadata;
import org.apache.druid.catalog.model.ColumnSpec;
import org.apache.druid.catalog.model.DatasourceBaseTableMetadata;
import org.apache.druid.catalog.model.DatasourceProjectionMetadata;
+import org.apache.druid.catalog.model.TableBaseTableMetadata;
import org.apache.druid.catalog.model.TableId;
import org.apache.druid.catalog.model.TableMetadata;
import org.apache.druid.catalog.model.TableSpec;
@@ -38,10 +39,15 @@ import org.apache.druid.catalog.model.table.ClusterKeySpec;
import org.apache.druid.catalog.model.table.DatasourceDefn;
import org.apache.druid.data.input.impl.AggregateProjectionSpec;
import org.apache.druid.error.DruidException;
+import org.apache.druid.guice.BuiltInTypesModule;
import org.apache.druid.java.util.common.JodaUtils;
import org.apache.druid.java.util.common.granularity.Granularities;
import org.apache.druid.query.filter.RangeFilter;
+import org.apache.druid.segment.VirtualColumn;
+import org.apache.druid.segment.VirtualColumns;
import org.apache.druid.segment.column.ColumnType;
+import org.apache.druid.segment.virtual.NestedFieldVirtualColumn;
+import org.apache.druid.segment.virtual.NestedMergeVirtualColumn;
import org.apache.druid.server.security.Action;
import org.apache.druid.server.security.AuthConfig;
import org.apache.druid.server.security.AuthenticationResult;
@@ -82,6 +88,12 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
@SqlTestFrameworkConfig.ComponentSupplier(CatalogDdlComponentSupplier.class)
public class CalciteCatalogDdlTest extends BaseCalciteQueryTest
{
+ static {
+ // Deriving a base-table spec resolves complex columns through the
dimension-handler registry, which the SQL test
+ // framework does not populate on its own.
+ BuiltInTypesModule.registerHandlersAndSerde();
+ }
+
private static final RecordingCatalogTableWriter WRITER = new
RecordingCatalogTableWriter();
public static class CatalogDdlComponentSupplier extends
StandardComponentSupplier
@@ -760,6 +772,62 @@ public class CalciteCatalogDdlTest extends
BaseCalciteQueryTest
);
}
+ /**
+ * A computed column keeps the virtual column type the planner produced for
it: {@code JSON_VALUE} plans to a
+ * nested-field virtual column, not a generic expression, and the stored
layout keeps that specialization (renamed to
+ * the declared column) so it matches the virtual columns the planner
generates for equivalent query expressions.
+ */
+ @Test
+ public void testBaseProjectionComputedColumnKeepsPlannedVirtualColumnType()
+ {
+ execute(
+ "CREATE TABLE tbl (tenant VARCHAR, payload TYPE('COMPLEX<json>'), v
BIGINT, __time TIMESTAMP,"
+ + " PROJECTION __base AS ("
+ + " SELECT tenant, payload, JSON_VALUE(payload, '$.v' RETURNING
BIGINT) AS v, __time"
+ + " CLUSTERED BY tenant"
+ + " ))"
+ );
+ assertEquals(
+ new ClusteredValueGroupsBaseTableMetadata(
+ ImmutableList.of("tenant"),
+ VirtualColumns.create(new NestedFieldVirtualColumn("payload",
"$.v", "v", ColumnType.LONG)),
+ null
+ ),
+
WRITER.calls.get(0).spec.properties().get(DatasourceDefn.BASE_TABLE_PROPERTY)
+ );
+ }
+
+ /**
+ * A computed column composed of specialized expressions lifts with its
dependency closure: the planner composes
+ * specializations by reference (the inner JSON_MERGE becomes its own
virtual column that the outer nested-field
+ * virtual column reads by synthetic name), so the intermediary rides into
the spec under its synthetic name as an
+ * unstored input of the materialized column, and both keep their planned
specialized types.
+ */
+ @Test
+ public void testBaseProjectionComposedExpressionLiftsDependencyClosure()
+ {
+ execute(
+ "CREATE TABLE tbl (tenant VARCHAR, payload TYPE('COMPLEX<json>'), v
BIGINT, __time TIMESTAMP,"
+ + " PROJECTION __base AS ("
+ + " SELECT tenant, payload, JSON_VALUE(JSON_MERGE(payload, payload),
'$.v' RETURNING BIGINT) AS v, __time"
+ + " CLUSTERED BY tenant"
+ + " ))"
+ );
+ final ClusteredValueGroupsBaseTableMetadata metadata =
+ (ClusteredValueGroupsBaseTableMetadata)
+
WRITER.calls.get(0).spec.properties().get(DatasourceDefn.BASE_TABLE_PROPERTY);
+ final VirtualColumns virtualColumns = metadata.getVirtualColumns();
+ assertEquals(2, virtualColumns.getVirtualColumns().length);
+
+ final VirtualColumn materialized = virtualColumns.getVirtualColumn("v");
+ assertTrue(materialized instanceof NestedFieldVirtualColumn,
String.valueOf(materialized));
+ assertEquals(1, materialized.requiredColumns().size());
+
+ final VirtualColumn intermediary =
virtualColumns.getVirtualColumn(materialized.requiredColumns().get(0));
+ assertTrue(intermediary instanceof NestedMergeVirtualColumn,
String.valueOf(intermediary));
+ assertEquals(ImmutableList.of("payload"), intermediary.requiredColumns());
+ }
+
/**
* A base table column is stored under the name it declares, so a body item
that renames another column is rejected:
* only a virtual column materializes a name the body did not select, and a
bare reference produces none.
@@ -835,6 +903,118 @@ public class CalciteCatalogDdlTest extends
BaseCalciteQueryTest
assertNull(spec.properties().get(DatasourceDefn.SEALED_PROPERTY));
}
+ /**
+ * A {@code __base} projection without {@code CLUSTERED BY} declares the
plain-table layout: declared column order is
+ * the physical storage and sort order, and no clustering structure is built.
+ */
+ @Test
+ public void testCreateTableWithPlainBaseProjection()
+ {
+ execute(
+ "CREATE TABLE tbl (tenant VARCHAR, __time TIMESTAMP, v BIGINT,"
+ + " PROJECTION __base AS (SELECT tenant, __time, v))"
+ );
+ final TableSpec spec = WRITER.calls.get(0).spec;
+ assertEquals(
+ new TableBaseTableMetadata(null, null),
+ spec.properties().get(DatasourceDefn.BASE_TABLE_PROPERTY)
+ );
+ }
+
+ /**
+ * The one expression a plain base table accepts: {@code TIME_FLOOR(__time,
<period>)} selected as {@code __time}
+ * declares the table's query granularity, stored as the spec's canonical
granularity-carrier virtual column rather
+ * than as a materialized column.
+ */
+ @Test
+ public void testPlainBaseProjectionEncodesQueryGranularity()
+ {
+ execute(
+ "CREATE TABLE tbl (tenant VARCHAR, __time TIMESTAMP, v BIGINT,"
+ + " PROJECTION __base AS (SELECT tenant, TIME_FLOOR(__time, 'PT1H') AS
__time, v))"
+ );
+ final TableSpec spec = WRITER.calls.get(0).spec;
+ assertEquals(
+ new TableBaseTableMetadata(
+ VirtualColumns.create(
+ Granularities.toVirtualColumn(Granularities.HOUR,
Granularities.GRANULARITY_VIRTUAL_COLUMN_NAME)
+ ),
+ null
+ ),
+ spec.properties().get(DatasourceDefn.BASE_TABLE_PROPERTY)
+ );
+ }
+
+ /**
+ * {@code __time} is stored in UTC, so a base table's query granularity must
be a UTC period: a session time zone
+ * reaching the body's TIME_FLOOR (the same way it reaches an aggregate
projection's) makes the granularity
+ * non-UTC, and the statement is rejected rather than storing buckets that
do not align with the stored values.
+ */
+ @Test
+ public void testPlainBaseProjectionRejectsNonUtcQueryGranularity()
+ {
+ final DruidException e = assertThrows(
+ DruidException.class,
+ () -> execute(
+ "SET sqlTimeZone = 'America/Los_Angeles';\n"
+ + "CREATE TABLE tbl (tenant VARCHAR, __time TIMESTAMP,"
+ + " PROJECTION __base AS (SELECT tenant, TIME_FLOOR(__time, 'P1D')
AS __time))"
+ )
+ );
+ assertTrue(
+ e.getMessage().contains("only period granularities in the UTC time
zone without an origin are supported"),
+ e.getMessage()
+ );
+ assertTrue(WRITER.calls.isEmpty());
+ }
+
+ /**
+ * The plain write path stores columns as they arrive, so a plain base table
rejects computed columns.
+ */
+ @Test
+ public void testPlainBaseProjectionRejectsComputedColumn()
+ {
+ final DruidException e = assertThrows(
+ DruidException.class,
+ () -> execute(
+ "CREATE TABLE tbl (tenant VARCHAR, bucket BIGINT, __time
TIMESTAMP,"
+ + " PROJECTION __base AS (SELECT tenant, ABS(bucket) AS bucket,
__time))"
+ )
+ );
+ assertTrue(
+ e.getMessage().contains(
+ "its column [bucket] is computed by an expression. Computed
columns are not supported; compute the column"
+ + " at ingestion time instead"
+ ),
+ e.getMessage()
+ );
+ assertTrue(WRITER.calls.isEmpty());
+ }
+
+ /**
+ * A computed {@code __time} is only meaningful as a granularity
declaration, so an expression the granularity layer
+ * cannot recover is rejected rather than silently stored as something else.
+ */
+ @Test
+ public void testPlainBaseProjectionRejectsNonGranularityTimeExpression()
+ {
+ final DruidException e = assertThrows(
+ DruidException.class,
+ () -> execute(
+ "CREATE TABLE tbl (tenant VARCHAR, __time TIMESTAMP,"
+ + " PROJECTION __base AS (SELECT tenant, TIME_SHIFT(__time,
'PT1H', 1) AS __time))"
+ )
+ );
+ assertTrue(
+ e.getMessage().contains(
+ "it computes [__time] with an expression that is not a
granularity. A base table computes [__time] only"
+ + " as TIME_FLOOR(__time, <period>), which declares the table's
query granularity"
+ ),
+ e.getMessage()
+ );
+ assertTrue(WRITER.calls.isEmpty());
+ }
+
/**
* The body lists the columns in the order segments store them, so it must
match the declaration exactly.
*/
diff --git
a/sql/src/test/java/org/apache/druid/sql/calcite/parser/DruidSqlDdlParserTest.java
b/sql/src/test/java/org/apache/druid/sql/calcite/parser/DruidSqlDdlParserTest.java
index 6f3afcc2cd5..cb2b24d43fc 100644
---
a/sql/src/test/java/org/apache/druid/sql/calcite/parser/DruidSqlDdlParserTest.java
+++
b/sql/src/test/java/org/apache/druid/sql/calcite/parser/DruidSqlDdlParserTest.java
@@ -456,6 +456,11 @@ public class DruidSqlDdlParserTest
"CREATE TABLE \"tbl\" (\"a\" VARCHAR, \"__time\" TIMESTAMP,"
+ " PROJECTION \"__base\" AS (SELECT \"a\", \"__time\" CLUSTERED BY
\"a\"))"
);
+ // Without CLUSTERED BY, the base projection declares the plain-table
layout.
+ assertUnparseRoundTrips(
+ "CREATE TABLE \"tbl\" (\"a\" VARCHAR, \"__time\" TIMESTAMP,"
+ + " PROJECTION \"__base\" AS (SELECT \"a\", \"__time\"))"
+ );
assertUnparseRoundTrips("ALTER TABLE \"tbl\" DROP PROJECTION \"p\"");
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
