This is an automated email from the ASF dual-hosted git repository.
xiangfu0 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new b0cb24b0e89 Add options-defined CREATE TABLE ... WITH (options) DDL
form with a pluggable handler (#18723)
b0cb24b0e89 is described below
commit b0cb24b0e89a352b0ad48ffa637fbf27504fc6bc
Author: Xiang Fu <[email protected]>
AuthorDate: Wed Jun 10 11:33:53 2026 -0700
Add options-defined CREATE TABLE ... WITH (options) DDL form with a
pluggable handler (#18723)
---
.../src/main/codegen/includes/parserImpls.ftl | 106 +++++++++-
.../sql/parsers/parser/SqlPinotCreateTable.java | 51 ++++-
.../pinot/sql/parsers/PinotDdlParserTest.java | 154 ++++++++++++++
.../ddl/compile/CreateTableWithOptionsHandler.java | 64 ++++++
.../apache/pinot/sql/ddl/compile/DdlCompiler.java | 100 ++++++++-
.../DefaultCreateTableWithOptionsHandler.java | 42 ++++
.../compile/CreateTableWithOptionsHandlerTest.java | 233 +++++++++++++++++++++
7 files changed, 735 insertions(+), 15 deletions(-)
diff --git a/pinot-common/src/main/codegen/includes/parserImpls.ftl
b/pinot-common/src/main/codegen/includes/parserImpls.ftl
index b0f1d3f798c..06070cff935 100644
--- a/pinot-common/src/main/codegen/includes/parserImpls.ftl
+++ b/pinot-common/src/main/codegen/includes/parserImpls.ftl
@@ -224,6 +224,15 @@ SqlLiteral PinotRefreshEveryPeriod() :
/// )
/// TABLE_TYPE = OFFLINE | REALTIME
/// [ PROPERTIES ( 'k' = 'v', ... ) ]
+///
+/// or the options-defined form, with no column list and no TABLE_TYPE:
+///
+/// CREATE TABLE [IF NOT EXISTS] [db.]name WITH ( key = value, ... )
+///
+/// In the options-defined form the schema and table config are derived
entirely from the WITH
+/// options by a pluggable handler in the DDL compiler (e.g. a handler that
connects to an
+/// external catalog named by the options and infers the schema from it). The
two forms are
+/// disambiguated by the token after the table name: `WITH` vs `(`.
SqlNode SqlPinotCreateTable() :
{
SqlParserPos pos;
@@ -233,21 +242,29 @@ SqlNode SqlPinotCreateTable() :
SqlNodeList primaryKeyColumns = null;
SqlLiteral tableType;
SqlNodeList properties = null;
+ SqlNodeList withOptions;
}
{
<CREATE> { pos = getPos(); }
<TABLE>
[ LOOKAHEAD(3) <IF> <NOT> <EXISTS> { ifNotExists = true; } ]
name = CompoundIdentifier()
- columns = PinotColumnList()
- [ LOOKAHEAD(2) primaryKeyColumns = PinotPrimaryKeyList() ]
- <TABLE_TYPE> <EQ>
- tableType = PinotTableTypeLiteral()
- [ <PROPERTIES> properties = PinotPropertyList() ]
- {
- return new SqlPinotCreateTable(pos, name, ifNotExists, columns,
primaryKeyColumns,
- tableType, properties);
- }
+ (
+ <WITH> withOptions = PinotWithOptionList()
+ {
+ return new SqlPinotCreateTable(pos, name, ifNotExists,
withOptions);
+ }
+ |
+ columns = PinotColumnList()
+ [ LOOKAHEAD(2) primaryKeyColumns = PinotPrimaryKeyList() ]
+ <TABLE_TYPE> <EQ>
+ tableType = PinotTableTypeLiteral()
+ [ <PROPERTIES> properties = PinotPropertyList() ]
+ {
+ return new SqlPinotCreateTable(pos, name, ifNotExists, columns,
primaryKeyColumns,
+ tableType, properties);
+ }
+ )
}
SqlNodeList PinotColumnList() :
@@ -368,6 +385,77 @@ SqlPinotProperty PinotProperty() :
}
}
+/// Parenthesized, comma-separated option list for the options-defined CREATE
TABLE form.
+/// Unlike PROPERTIES (whose keys and values are always quoted string
literals), WITH options
+/// accept the option shape common to external-table DDL in other engines:
unquoted — possibly
+/// dotted — identifier keys and typed literal values, e.g. `catalog_type =
'rest'`,
+/// `storage.region = 'us-west-2'`, `enable_schema_evolution = true`. Quoted
keys remain
+/// accepted, so the PROPERTIES style also works. Keys and values are
normalized to character
+/// literals at parse time so downstream consumers uniformly see string pairs
+/// ([SqlPinotProperty]).
+SqlNodeList PinotWithOptionList() :
+{
+ SqlParserPos pos;
+ SqlPinotProperty option;
+ List<SqlNode> list = new ArrayList<SqlNode>();
+}
+{
+ <LPAREN> { pos = getPos(); }
+ [
+ option = PinotWithOption() { list.add(option); }
+ ( <COMMA> option = PinotWithOption() { list.add(option); } )*
+ ]
+ <RPAREN>
+ {
+ return new SqlNodeList(list, pos.plus(getPos()));
+ }
+}
+
+/// A single `key = value` WITH option. Key: quoted string literal or (dotted)
compound
+/// identifier — identifier keys are case-preserved (the parser runs with
unquoted casing
+/// UNCHANGED) and joined with `.`. Value: quoted string literal, TRUE /
FALSE, or an unsigned
+/// numeric literal.
+SqlPinotProperty PinotWithOption() :
+{
+ SqlParserPos pos;
+ SqlNode keyNode;
+ SqlIdentifier keyId;
+ SqlLiteral key;
+ SqlNode valueNode;
+ SqlLiteral value;
+}
+{
+ (
+ keyNode = StringLiteral()
+ {
+ pos = getPos();
+ key = (SqlLiteral) keyNode;
+ }
+ |
+ keyId = CompoundIdentifier()
+ {
+ pos = getPos();
+ key = SqlLiteral.createCharString(String.join(".", keyId.names),
keyId.getParserPosition());
+ }
+ )
+ <EQ>
+ (
+ valueNode = StringLiteral() { value = (SqlLiteral) valueNode; }
+ |
+ <TRUE> { value = SqlLiteral.createCharString("true", getPos()); }
+ |
+ <FALSE> { value = SqlLiteral.createCharString("false", getPos()); }
+ |
+ valueNode = UnsignedNumericLiteral()
+ {
+ value = SqlLiteral.createCharString(((SqlLiteral)
valueNode).toValue(), getPos());
+ }
+ )
+ {
+ return new SqlPinotProperty(pos, key, value);
+ }
+}
+
/// DROP TABLE [IF EXISTS] [db.]name [TYPE OFFLINE | REALTIME]
/// | DROP MATERIALIZED VIEW [IF EXISTS] [db.]name
///
diff --git
a/pinot-common/src/main/java/org/apache/pinot/sql/parsers/parser/SqlPinotCreateTable.java
b/pinot-common/src/main/java/org/apache/pinot/sql/parsers/parser/SqlPinotCreateTable.java
index 9295575eb5d..902467e2831 100644
---
a/pinot-common/src/main/java/org/apache/pinot/sql/parsers/parser/SqlPinotCreateTable.java
+++
b/pinot-common/src/main/java/org/apache/pinot/sql/parsers/parser/SqlPinotCreateTable.java
@@ -35,7 +35,7 @@ import org.apache.calcite.sql.parser.SqlParserPos;
/// Pinot-native `CREATE TABLE` DDL statement.
///
-/// Syntax:
+/// Syntax (column-list form):
/// ```
/// CREATE TABLE [IF NOT EXISTS] [db.]name (
/// col TYPE [NULL | NOT NULL] [DEFAULT literal] [DIMENSION | METRIC],
@@ -50,6 +50,19 @@ import org.apache.calcite.sql.parser.SqlParserPos;
/// )
/// ```
///
+/// or the options-defined form, where the schema and table config are derived
entirely from the
+/// WITH options by a pluggable handler in the DDL compiler:
+/// ```
+/// CREATE TABLE [IF NOT EXISTS] [db.]name WITH (
+/// key = value,
+/// ...
+/// )
+/// ```
+///
+/// The two forms are mutually exclusive: [#getWithOptions] is non-null
exactly for the
+/// options-defined form, in which case [#getColumns] / [#getProperties] are
empty and
+/// [#getTableType] / [#getPrimaryKeyColumns] are null.
+///
/// This is a parse-time AST node only. Semantic validation (data type
recognition, role
/// inference, property mapping) happens in `DdlCompiler` in the
`pinot-sql-ddl` module.
///
@@ -62,11 +75,23 @@ public class SqlPinotCreateTable extends SqlCall {
private final boolean _ifNotExists;
private final SqlNodeList _columns;
@Nullable private final SqlNodeList _primaryKeyColumns;
- private final SqlLiteral _tableType;
+ @Nullable private final SqlLiteral _tableType;
private final SqlNodeList _properties;
+ @Nullable private final SqlNodeList _withOptions;
public SqlPinotCreateTable(SqlParserPos pos, SqlIdentifier name, boolean
ifNotExists, SqlNodeList columns,
@Nullable SqlNodeList primaryKeyColumns, SqlLiteral tableType, @Nullable
SqlNodeList properties) {
+ this(pos, name, ifNotExists, columns, primaryKeyColumns, tableType,
properties, null);
+ }
+
+ /// Options-defined form: `CREATE TABLE [IF NOT EXISTS] [db.]name WITH (key
= value, ...)`.
+ public SqlPinotCreateTable(SqlParserPos pos, SqlIdentifier name, boolean
ifNotExists, SqlNodeList withOptions) {
+ this(pos, name, ifNotExists, SqlNodeList.EMPTY, null, null, null,
withOptions);
+ }
+
+ private SqlPinotCreateTable(SqlParserPos pos, SqlIdentifier name, boolean
ifNotExists, SqlNodeList columns,
+ @Nullable SqlNodeList primaryKeyColumns, @Nullable SqlLiteral tableType,
@Nullable SqlNodeList properties,
+ @Nullable SqlNodeList withOptions) {
super(pos);
_name = name;
_ifNotExists = ifNotExists;
@@ -74,6 +99,7 @@ public class SqlPinotCreateTable extends SqlCall {
_primaryKeyColumns = primaryKeyColumns;
_tableType = tableType;
_properties = properties == null ? SqlNodeList.EMPTY : properties;
+ _withOptions = withOptions;
}
public SqlIdentifier getName() {
@@ -93,6 +119,8 @@ public class SqlPinotCreateTable extends SqlCall {
return _primaryKeyColumns;
}
+ /// Null only for the options-defined (`WITH`) form.
+ @Nullable
public SqlLiteral getTableType() {
return _tableType;
}
@@ -101,6 +129,13 @@ public class SqlPinotCreateTable extends SqlCall {
return _properties;
}
+ /// The `WITH (key = value, ...)` option list, or null for the column-list
form. Non-null (even
+ /// when empty) means the statement was written in the options-defined form.
+ @Nullable
+ public SqlNodeList getWithOptions() {
+ return _withOptions;
+ }
+
@Override
public SqlOperator getOperator() {
return OPERATOR;
@@ -109,7 +144,7 @@ public class SqlPinotCreateTable extends SqlCall {
@Override
public List<SqlNode> getOperandList() {
// Fixed-arity list with null placeholder for absent optional operand, per
Calcite SqlCall contract.
- return Arrays.asList(_name, _columns, _primaryKeyColumns, _tableType,
_properties);
+ return Arrays.asList(_name, _columns, _primaryKeyColumns, _tableType,
_properties, _withOptions);
}
@Override
@@ -119,6 +154,16 @@ public class SqlPinotCreateTable extends SqlCall {
writer.keyword("IF NOT EXISTS");
}
_name.unparse(writer, leftPrec, rightPrec);
+ if (_withOptions != null) {
+ writer.keyword("WITH");
+ SqlWriter.Frame withFrame = writer.startList("(", ")");
+ for (SqlNode option : _withOptions) {
+ writer.sep(",");
+ option.unparse(writer, 0, 0);
+ }
+ writer.endList(withFrame);
+ return;
+ }
SqlWriter.Frame columnFrame = writer.startList("(", ")");
for (SqlNode column : _columns) {
writer.sep(",");
diff --git
a/pinot-common/src/test/java/org/apache/pinot/sql/parsers/PinotDdlParserTest.java
b/pinot-common/src/test/java/org/apache/pinot/sql/parsers/PinotDdlParserTest.java
index 1f6ca48d319..db2b2fd5f0a 100644
---
a/pinot-common/src/test/java/org/apache/pinot/sql/parsers/PinotDdlParserTest.java
+++
b/pinot-common/src/test/java/org/apache/pinot/sql/parsers/PinotDdlParserTest.java
@@ -21,6 +21,7 @@ package org.apache.pinot.sql.parsers;
import java.util.Locale;
import org.apache.calcite.sql.SqlIdentifier;
import org.apache.calcite.sql.SqlNode;
+import org.apache.calcite.sql.dialect.CalciteSqlDialect;
import org.apache.pinot.sql.parsers.parser.SqlPinotColumnDeclaration;
import org.apache.pinot.sql.parsers.parser.SqlPinotCreateMaterializedView;
import org.apache.pinot.sql.parsers.parser.SqlPinotCreateTable;
@@ -154,6 +155,159 @@ public class PinotDdlParserTest {
assertEquals(node.getProperties().size(), 3);
}
+ //
-------------------------------------------------------------------------------------------
+ // CREATE TABLE ... WITH (options) — the options-defined form
+ //
-------------------------------------------------------------------------------------------
+
+ @Test
+ public void createTableWithOptionsMinimal() {
+ SqlPinotCreateTable node = parseCreate("CREATE TABLE trips WITH ('type' =
'iceberg')");
+ assertNotNull(node.getWithOptions());
+ assertEquals(node.getWithOptions().size(), 1);
+ SqlPinotProperty option = (SqlPinotProperty) node.getWithOptions().get(0);
+ assertEquals(option.getKeyString(), "type");
+ assertEquals(option.getValueString(), "iceberg");
+ // The options-defined form carries no column-list-form clauses.
+ assertTrue(node.getColumns().isEmpty());
+ assertNull(node.getPrimaryKeyColumns());
+ assertNull(node.getTableType());
+ assertTrue(node.getProperties().isEmpty());
+ }
+
+ @Test
+ public void classicCreateTableHasNullWithOptions() {
+ // getWithOptions() is the form discriminator: null means column-list
form, non-null (even
+ // empty) means the WITH form was written.
+ SqlPinotCreateTable node = parseCreate("CREATE TABLE events (id INT)
TABLE_TYPE = OFFLINE");
+ assertNull(node.getWithOptions());
+ }
+
+ @Test
+ public void createTableWithOptionsFullIcebergSample() {
+ // The motivating external-table DDL shape: unquoted snake_case keys,
dotted keys, string /
+ // boolean values, and ${...} placeholder values passed through verbatim.
+ SqlPinotCreateTable node = parseCreate(
+ "CREATE TABLE trips_analytics WITH ("
+ + " type = 'iceberg',"
+ + " catalog_type = 'rest',"
+ + " catalog_uri =
'https://unity-catalog.company.com/api/2.1/unity-catalog/iceberg',"
+ + " catalog_name = 'main',"
+ + " schema_name = 'transportation',"
+ + " table_name = 'nyc_taxi_trips',"
+ + " auth_type = 'oauth',"
+ + " client_id = '${UC_CLIENT_ID}',"
+ + " client_secret = '${UC_CLIENT_SECRET}',"
+ + " oauth_scope = 'all-apis',"
+ + " storage.provider = 's3',"
+ + " storage.region = 'us-west-2',"
+ + " storage.access_key = '${AWS_ACCESS_KEY_ID}',"
+ + " storage.secret_key = '${AWS_SECRET_ACCESS_KEY}',"
+ + " refresh_interval = '5m',"
+ + " snapshot_mode = 'latest',"
+ + " enable_schema_evolution = true,"
+ + " enable_partition_pruning = true,"
+ + " enable_metadata_caching = true"
+ + ")");
+ assertEquals(node.getName().getSimple(), "trips_analytics");
+ assertNotNull(node.getWithOptions());
+ assertEquals(node.getWithOptions().size(), 19);
+
+ SqlPinotProperty type = (SqlPinotProperty) node.getWithOptions().get(0);
+ assertEquals(type.getKeyString(), "type");
+ assertEquals(type.getValueString(), "iceberg");
+
+ // Unquoted identifier keys are case-preserved verbatim.
+ SqlPinotProperty catalogType = (SqlPinotProperty)
node.getWithOptions().get(1);
+ assertEquals(catalogType.getKeyString(), "catalog_type");
+ assertEquals(catalogType.getValueString(), "rest");
+
+ // ${...} placeholders are opaque string values; no substitution at parse
time.
+ SqlPinotProperty clientId = (SqlPinotProperty)
node.getWithOptions().get(7);
+ assertEquals(clientId.getKeyString(), "client_id");
+ assertEquals(clientId.getValueString(), "${UC_CLIENT_ID}");
+
+ // Dotted identifier keys join with '.'.
+ SqlPinotProperty storageProvider = (SqlPinotProperty)
node.getWithOptions().get(10);
+ assertEquals(storageProvider.getKeyString(), "storage.provider");
+ assertEquals(storageProvider.getValueString(), "s3");
+
+ // Boolean values normalize to lower-case string form.
+ SqlPinotProperty schemaEvolution = (SqlPinotProperty)
node.getWithOptions().get(16);
+ assertEquals(schemaEvolution.getKeyString(), "enable_schema_evolution");
+ assertEquals(schemaEvolution.getValueString(), "true");
+ }
+
+ @Test
+ public void createTableWithOptionsIfNotExistsAndQualifiedName() {
+ SqlPinotCreateTable node = parseCreate(
+ "CREATE TABLE IF NOT EXISTS myDb.trips WITH ('type' = 'iceberg')");
+ assertTrue(node.isIfNotExists());
+ assertEquals(node.getName().names.size(), 2);
+ assertEquals(node.getName().names.get(0), "myDb");
+ assertEquals(node.getName().names.get(1), "trips");
+ }
+
+ @Test
+ public void createTableWithOptionsKeyQuotingAndValueTypes() {
+ // Quoted and unquoted keys are interchangeable; values may be strings,
booleans, or numbers
+ // — all normalized to plain string pairs.
+ SqlPinotCreateTable node = parseCreate(
+ "CREATE TABLE t WITH ("
+ + " 'quoted.key' = 'v1',"
+ + " unquoted_key = 'v2',"
+ + " bool_key = false,"
+ + " int_key = 100,"
+ + " decimal_key = 2.5"
+ + ")");
+ assertNotNull(node.getWithOptions());
+ assertEquals(node.getWithOptions().size(), 5);
+ assertEquals(((SqlPinotProperty)
node.getWithOptions().get(0)).getKeyString(), "quoted.key");
+ assertEquals(((SqlPinotProperty)
node.getWithOptions().get(1)).getKeyString(), "unquoted_key");
+ assertEquals(((SqlPinotProperty)
node.getWithOptions().get(2)).getValueString(), "false");
+ assertEquals(((SqlPinotProperty)
node.getWithOptions().get(3)).getValueString(), "100");
+ assertEquals(((SqlPinotProperty)
node.getWithOptions().get(4)).getValueString(), "2.5");
+ }
+
+ @Test
+ public void createTableWithOptionsEmptyListParses() {
+ // Parse-level permissive (mirrors PROPERTIES ()); the DDL compiler
rejects empty options.
+ SqlPinotCreateTable node = parseCreate("CREATE TABLE t WITH ()");
+ assertNotNull(node.getWithOptions());
+ assertTrue(node.getWithOptions().isEmpty());
+ }
+
+ @Test
+ public void createTableWithOptionsUnparseRoundTrips() {
+ SqlPinotCreateTable node = parseCreate(
+ "CREATE TABLE trips WITH (type = 'iceberg', storage.region =
'us-west-2', enabled = true)");
+ // Unparse emits the canonical quoted-key form; re-parsing it must
preserve the options. The
+ // Calcite dialect double-quotes identifiers, matching the parser's DQID
lexical state.
+ SqlPinotCreateTable reparsed =
parseCreate(node.toSqlString(CalciteSqlDialect.DEFAULT).getSql());
+ assertNotNull(reparsed.getWithOptions());
+ assertEquals(reparsed.getWithOptions().size(), 3);
+ SqlPinotProperty type = (SqlPinotProperty)
reparsed.getWithOptions().get(0);
+ assertEquals(type.getKeyString(), "type");
+ assertEquals(type.getValueString(), "iceberg");
+ SqlPinotProperty region = (SqlPinotProperty)
reparsed.getWithOptions().get(1);
+ assertEquals(region.getKeyString(), "storage.region");
+ assertEquals(region.getValueString(), "us-west-2");
+ SqlPinotProperty enabled = (SqlPinotProperty)
reparsed.getWithOptions().get(2);
+ assertEquals(enabled.getKeyString(), "enabled");
+ assertEquals(enabled.getValueString(), "true");
+ }
+
+ @Test
+ public void createTableWithOptionsRejectsColumnListFormClauses() {
+ // The two forms are mutually exclusive: no TABLE_TYPE / PROPERTIES after
WITH (...), and no
+ // WITH (...) after a column list.
+ expectThrows(SqlCompilationException.class,
+ () -> CalciteSqlParser.compileToSqlNodeAndOptions(
+ "CREATE TABLE t WITH ('k' = 'v') TABLE_TYPE = OFFLINE"));
+ expectThrows(SqlCompilationException.class,
+ () -> CalciteSqlParser.compileToSqlNodeAndOptions(
+ "CREATE TABLE t (id INT) WITH ('k' = 'v')"));
+ }
+
//
-------------------------------------------------------------------------------------------
// CREATE MATERIALIZED VIEW
//
-------------------------------------------------------------------------------------------
diff --git
a/pinot-sql-ddl/src/main/java/org/apache/pinot/sql/ddl/compile/CreateTableWithOptionsHandler.java
b/pinot-sql-ddl/src/main/java/org/apache/pinot/sql/ddl/compile/CreateTableWithOptionsHandler.java
new file mode 100644
index 00000000000..540f9f57cc9
--- /dev/null
+++
b/pinot-sql-ddl/src/main/java/org/apache/pinot/sql/ddl/compile/CreateTableWithOptionsHandler.java
@@ -0,0 +1,64 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.sql.ddl.compile;
+
+import java.util.Map;
+import javax.annotation.Nullable;
+
+
+/// Extension point for compiling the options-defined `CREATE TABLE ... WITH
(key = value, ...)`
+/// form — a `CREATE TABLE` with no column list whose schema and table config
are derived entirely
+/// from the WITH options.
+///
+/// OSS Pinot has no built-in options-defined table type, so the default
implementation
+/// ([DefaultCreateTableWithOptionsHandler]) rejects the form with an
actionable error. A
+/// distribution that supports options-defined tables installs its own handler
via
+/// [DdlCompiler#setCreateTableWithOptionsHandler] once at controller startup,
before any DDL is
+/// served — e.g. a handler that recognizes `type = 'iceberg'`, connects to
the external catalog
+/// named by the options, infers the Pinot schema from the catalog table, and
emits a
+/// `TableConfig` wired to that distribution's ingestion machinery.
+///
+/// The returned [CompiledCreateTable] flows through the same controller
execution path as the
+/// column-list form (authorization, validation, schema + table-config
persistence, `IF NOT
+/// EXISTS` semantics), so the handler is responsible only for compilation —
never persistence.
+///
+/// Implementations must be thread-safe: a single process-wide instance serves
concurrent DDL
+/// requests.
+public interface CreateTableWithOptionsHandler {
+
+ /// Compiles one options-defined CREATE TABLE statement into a schema +
table config.
+ ///
+ /// @param databaseName database from the qualified table name, or null when
unqualified (the
+ /// controller resolves the effective database at
execution time)
+ /// @param tableName raw table name, without database prefix or table-type
suffix
+ /// @param ifNotExists whether the statement carries `IF NOT EXISTS`; echo
it into the returned
+ /// [CompiledCreateTable] — the controller implements the
semantics
+ /// @param options the WITH options in declaration order; keys are
case-preserved and unique
+ /// (case-insensitive duplicates are rejected before this
call), values are the
+ /// literal strings as written (no placeholder or environment
substitution)
+ /// @param ctx side-channel collaborators (table cache, request database);
never null, but may
+ /// be [DdlCompileContext#STATELESS] for callers without cluster
state
+ /// @return the compiled schema + table config; never null. The statement
identity is fixed by
+ /// the SQL text: the returned database name, raw table-config name,
schema name, and
+ /// `IF NOT EXISTS` flag must echo the supplied arguments —
[DdlCompiler] rejects any
+ /// mismatch, since downstream authorization and persistence act on
the returned names.
+ /// @throws DdlCompilationException when the options are invalid or the form
is unsupported
+ CompiledCreateTable compile(@Nullable String databaseName, String tableName,
boolean ifNotExists,
+ Map<String, String> options, DdlCompileContext ctx);
+}
diff --git
a/pinot-sql-ddl/src/main/java/org/apache/pinot/sql/ddl/compile/DdlCompiler.java
b/pinot-sql-ddl/src/main/java/org/apache/pinot/sql/ddl/compile/DdlCompiler.java
index 0a4b791fc5b..97d919d8586 100644
---
a/pinot-sql-ddl/src/main/java/org/apache/pinot/sql/ddl/compile/DdlCompiler.java
+++
b/pinot-sql-ddl/src/main/java/org/apache/pinot/sql/ddl/compile/DdlCompiler.java
@@ -24,6 +24,7 @@ import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
+import java.util.Objects;
import java.util.Set;
import javax.annotation.Nullable;
import org.apache.calcite.rel.type.RelDataType;
@@ -44,6 +45,7 @@ import org.apache.pinot.spi.data.MetricFieldSpec;
import org.apache.pinot.spi.data.Schema;
import org.apache.pinot.spi.utils.CommonConstants.MaterializedViewTask;
import org.apache.pinot.spi.utils.builder.TableConfigBuilder;
+import org.apache.pinot.spi.utils.builder.TableNameBuilder;
import org.apache.pinot.sql.ddl.inferer.MaterializedViewInferenceInput;
import org.apache.pinot.sql.ddl.inferer.MaterializedViewSchemaInferer;
import org.apache.pinot.sql.ddl.resolved.ColumnRole;
@@ -96,13 +98,27 @@ public final class DdlCompiler {
private static volatile MaterializedViewDdlHandler
_materializedViewDdlHandler =
new DefaultMaterializedViewDdlHandler();
+ /// Pluggable handler for the options-defined `CREATE TABLE ... WITH (key =
value, ...)` form
+ /// (no column list — the schema and table config are derived from the
options, e.g. by
+ /// connecting to an external catalog named by the options). OSS Pinot has
no built-in
+ /// options-defined table type, so the default rejects the form with an
actionable error; a
+ /// distribution that supports it installs its own handler via
+ /// [#setCreateTableWithOptionsHandler] once at controller startup, before
any DDL is served.
+ /// Volatile for the same reason as [#_materializedViewDdlHandler].
+ private static volatile CreateTableWithOptionsHandler
_createTableWithOptionsHandler =
+ new DefaultCreateTableWithOptionsHandler();
+
private DdlCompiler() {
}
/// Installs the [MaterializedViewDdlHandler] used for all subsequent
`CREATE MATERIALIZED VIEW`
/// compilations. Call once at controller startup; defaults to
[DefaultMaterializedViewDdlHandler].
+ /// The handler must not be null — the getter guarantees never-null, so a
null installation
+ /// fails fast here instead of surfacing as a confusing NPE on the next DDL
compilation.
public static void setMaterializedViewDdlHandler(MaterializedViewDdlHandler
handler) {
- _materializedViewDdlHandler = handler;
+ _materializedViewDdlHandler = Objects.requireNonNull(handler,
+ "MaterializedViewDdlHandler must not be null; to restore default
behavior install a new "
+ + "DefaultMaterializedViewDdlHandler instead");
}
/// Returns the active materialized-view DDL handler (never null; defaults
to single-source SSE).
@@ -110,6 +126,22 @@ public final class DdlCompiler {
return _materializedViewDdlHandler;
}
+ /// Installs the [CreateTableWithOptionsHandler] used for all subsequent
options-defined
+ /// `CREATE TABLE ... WITH (...)` compilations. Call once at controller
startup; defaults to
+ /// [DefaultCreateTableWithOptionsHandler], which rejects the form.
+ /// The handler must not be null — the getter guarantees never-null, so a
null installation
+ /// fails fast here instead of surfacing as a confusing NPE on the next DDL
compilation.
+ public static void
setCreateTableWithOptionsHandler(CreateTableWithOptionsHandler handler) {
+ _createTableWithOptionsHandler = Objects.requireNonNull(handler,
+ "CreateTableWithOptionsHandler must not be null; to restore default
behavior install a "
+ + "new DefaultCreateTableWithOptionsHandler instead");
+ }
+
+ /// Returns the active options-defined CREATE TABLE handler (never null;
defaults to rejecting).
+ public static CreateTableWithOptionsHandler
getCreateTableWithOptionsHandler() {
+ return _createTableWithOptionsHandler;
+ }
+
/// Parses and compiles a DDL statement using a stateless {@link
DdlCompileContext}.
///
/// @deprecated Use {@link #compile(String, DdlCompileContext)} and supply a
real context.
@@ -157,7 +189,7 @@ public final class DdlCompiler {
return compileShowMaterializedViews((SqlPinotShowMaterializedViews)
node);
}
if (node instanceof SqlPinotCreateTable) {
- return compileCreate((SqlPinotCreateTable) node);
+ return compileCreate((SqlPinotCreateTable) node, ctx);
}
if (node instanceof SqlPinotShowCreateTable) {
return compileShowCreate((SqlPinotShowCreateTable) node);
@@ -216,7 +248,10 @@ public final class DdlCompiler {
// Table DDL: CREATE TABLE
//
-------------------------------------------------------------------------------------------
- private static CompiledCreateTable compileCreate(SqlPinotCreateTable node) {
+ private static CompiledCreateTable compileCreate(SqlPinotCreateTable node,
DdlCompileContext ctx) {
+ if (node.getWithOptions() != null) {
+ return compileCreateWithOptions(node, ctx);
+ }
QualifiedName name = parseQualifiedName(node.getName());
TableType tableType = parseTableType(node.getTableType().toValue());
@@ -261,6 +296,65 @@ public final class DdlCompiler {
resolved.isIfNotExists(), warnings);
}
+ /// Compiles the options-defined `CREATE TABLE ... WITH (key = value, ...)`
form by delegating
+ /// to the registered [CreateTableWithOptionsHandler]. The compiler owns the
parts common to
+ /// both CREATE TABLE forms — name/database resolution and option
de-duplication — and the
+ /// handler owns everything the options mean (schema derivation,
table-config construction).
+ private static CompiledCreateTable
compileCreateWithOptions(SqlPinotCreateTable node, DdlCompileContext ctx) {
+ QualifiedName name = parseQualifiedName(node.getName());
+ Map<String, String> options =
resolveProperties(node.getWithOptions().getList());
+ if (options.isEmpty()) {
+ throw new DdlCompilationException("CREATE TABLE ... WITH requires at
least one option.");
+ }
+ CompiledCreateTable compiled = _createTableWithOptionsHandler.compile(
+ name._databaseName, name._tableName, node.isIfNotExists(), options,
ctx);
+ if (compiled == null) {
+ throw new DdlCompilationException(
+ "The options-defined CREATE TABLE handler returned no result for
table: " + name._tableName);
+ }
+ return validateHandlerResult(compiled, name, node.isIfNotExists());
+ }
+
+ /// The statement identity comes from the SQL text, not from the handler:
downstream
+ /// authorization and persistence act on the names carried by the returned
+ /// [CompiledCreateTable], so a buggy or malicious handler must not be able
to redirect the
+ /// DDL to a different database/table than the one the user named (and
authorization was
+ /// checked against). Rejects any mismatch instead of silently correcting
it, so handler bugs
+ /// surface immediately.
+ private static CompiledCreateTable validateHandlerResult(CompiledCreateTable
compiled,
+ QualifiedName name, boolean ifNotExists) {
+ if (compiled.getSchema() == null || compiled.getTableConfig() == null) {
+ throw new DdlCompilationException(
+ "The options-defined CREATE TABLE handler returned a result without
a "
+ + (compiled.getSchema() == null ? "schema" : "table config")
+ + " for table: " + name._tableName);
+ }
+ if (!Objects.equals(compiled.getDatabaseName(), name._databaseName)) {
+ throw new DdlCompilationException(
+ "The options-defined CREATE TABLE handler changed the database name:
expected '"
+ + name._databaseName + "' from the SQL text, got '" +
compiled.getDatabaseName() + "'.");
+ }
+ String rawTableName =
+
TableNameBuilder.extractRawTableName(compiled.getTableConfig().getTableName());
+ if (!name._tableName.equals(rawTableName)) {
+ throw new DdlCompilationException(
+ "The options-defined CREATE TABLE handler changed the table name:
expected '"
+ + name._tableName + "' from the SQL text, got '" + rawTableName
+ "'.");
+ }
+ if (!name._tableName.equals(compiled.getSchema().getSchemaName())) {
+ throw new DdlCompilationException(
+ "The options-defined CREATE TABLE handler returned a schema named '"
+ + compiled.getSchema().getSchemaName() + "'; the schema name
must match the table"
+ + " name from the SQL text: '" + name._tableName + "'.");
+ }
+ if (compiled.isIfNotExists() != ifNotExists) {
+ throw new DdlCompilationException(
+ "The options-defined CREATE TABLE handler changed the IF NOT EXISTS
flag: expected "
+ + ifNotExists + " from the SQL text, got " +
compiled.isIfNotExists() + ".");
+ }
+ return compiled;
+ }
+
private static List<ResolvedColumnDefinition> resolveColumns(List<SqlNode>
columnNodes,
List<String> warnings) {
if (columnNodes.isEmpty()) {
diff --git
a/pinot-sql-ddl/src/main/java/org/apache/pinot/sql/ddl/compile/DefaultCreateTableWithOptionsHandler.java
b/pinot-sql-ddl/src/main/java/org/apache/pinot/sql/ddl/compile/DefaultCreateTableWithOptionsHandler.java
new file mode 100644
index 00000000000..58ef2372a30
--- /dev/null
+++
b/pinot-sql-ddl/src/main/java/org/apache/pinot/sql/ddl/compile/DefaultCreateTableWithOptionsHandler.java
@@ -0,0 +1,42 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.sql.ddl.compile;
+
+import java.util.Map;
+import javax.annotation.Nullable;
+
+
+/// Default [CreateTableWithOptionsHandler]. OSS Pinot has no built-in
options-defined table
+/// type, so the `CREATE TABLE ... WITH (options)` form is rejected with
guidance toward the
+/// column-list form. A distribution that supports options-defined tables
(e.g. external tables
+/// backed by an Iceberg catalog) replaces this handler via
+/// [DdlCompiler#setCreateTableWithOptionsHandler].
+///
+/// Stateless and thread-safe.
+public class DefaultCreateTableWithOptionsHandler implements
CreateTableWithOptionsHandler {
+
+ @Override
+ public CompiledCreateTable compile(@Nullable String databaseName, String
tableName, boolean ifNotExists,
+ Map<String, String> options, DdlCompileContext ctx) {
+ throw new DdlCompilationException(
+ "CREATE TABLE ... WITH (options) is not supported: no options-defined
table handler is "
+ + "installed in this distribution. Use the column-list form
instead: CREATE TABLE "
+ + tableName + " (col TYPE, ...) TABLE_TYPE = OFFLINE | REALTIME
[PROPERTIES (...)].");
+ }
+}
diff --git
a/pinot-sql-ddl/src/test/java/org/apache/pinot/sql/ddl/compile/CreateTableWithOptionsHandlerTest.java
b/pinot-sql-ddl/src/test/java/org/apache/pinot/sql/ddl/compile/CreateTableWithOptionsHandlerTest.java
new file mode 100644
index 00000000000..586f73daff3
--- /dev/null
+++
b/pinot-sql-ddl/src/test/java/org/apache/pinot/sql/ddl/compile/CreateTableWithOptionsHandlerTest.java
@@ -0,0 +1,233 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.sql.ddl.compile;
+
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import javax.annotation.Nullable;
+import org.apache.pinot.spi.config.table.TableConfig;
+import org.apache.pinot.spi.config.table.TableType;
+import org.apache.pinot.spi.data.Schema;
+import org.apache.pinot.spi.utils.builder.TableConfigBuilder;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.Test;
+
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertFalse;
+import static org.testng.Assert.assertNull;
+import static org.testng.Assert.assertSame;
+import static org.testng.Assert.assertTrue;
+import static org.testng.Assert.expectThrows;
+
+
+/// Tests the [CreateTableWithOptionsHandler] extension point: the default
handler rejects the
+/// options-defined `CREATE TABLE ... WITH (...)` form with actionable
guidance, while a
+/// registered handler receives the parsed name / flags / options and owns the
compilation.
+public class CreateTableWithOptionsHandlerTest {
+
+ @AfterMethod
+ public void restoreDefaultHandler() {
+ /// The handler is process-wide; restore the default so sibling tests see
rejecting behavior.
+ DdlCompiler.setCreateTableWithOptionsHandler(new
DefaultCreateTableWithOptionsHandler());
+ }
+
+ @Test
+ public void defaultsToRejectingHandler() {
+ assertTrue(DdlCompiler.getCreateTableWithOptionsHandler() instanceof
DefaultCreateTableWithOptionsHandler);
+ }
+
+ @Test
+ public void nullHandlerInstallationFailsFast() {
+ // The getters are documented never-null; a null installation must fail at
the setter rather
+ // than as a confusing NPE on the next DDL compilation.
+ expectThrows(NullPointerException.class,
+ () -> DdlCompiler.setCreateTableWithOptionsHandler(null));
+ expectThrows(NullPointerException.class,
+ () -> DdlCompiler.setMaterializedViewDdlHandler(null));
+ // The previously installed handlers survive the rejected installation.
+ assertTrue(DdlCompiler.getCreateTableWithOptionsHandler() instanceof
DefaultCreateTableWithOptionsHandler);
+ }
+
+ @Test
+ public void defaultHandlerRejectsWithGuidance() {
+ DdlCompilationException e = expectThrows(DdlCompilationException.class,
+ () -> DdlCompiler.compile("CREATE TABLE trips WITH (type =
'iceberg')", DdlCompileContext.STATELESS));
+ assertTrue(e.getMessage().contains("not supported"),
+ "Expected 'not supported' in: " + e.getMessage());
+ assertTrue(e.getMessage().contains("column-list form"),
+ "Expected guidance toward the column-list form in: " + e.getMessage());
+ }
+
+ @Test
+ public void registeredHandlerReceivesParsedStatement() {
+ RecordingHandler handler = new RecordingHandler();
+ DdlCompiler.setCreateTableWithOptionsHandler(handler);
+
+ CompiledDdl compiled = DdlCompiler.compile(
+ "CREATE TABLE IF NOT EXISTS myDb.trips WITH ("
+ + " type = 'iceberg',"
+ + " catalog_type = 'rest',"
+ + " storage.region = 'us-west-2',"
+ + " enable_schema_evolution = true"
+ + ")",
+ DdlCompileContext.STATELESS);
+
+ assertEquals(handler._databaseName, "myDb");
+ assertEquals(handler._tableName, "trips");
+ assertTrue(handler._ifNotExists);
+ // Options arrive in declaration order with case-preserved keys and
stringified values.
+ assertEquals(new ArrayList<>(handler._options.keySet()),
+ List.of("type", "catalog_type", "storage.region",
"enable_schema_evolution"));
+ assertEquals(handler._options.get("type"), "iceberg");
+ assertEquals(handler._options.get("enable_schema_evolution"), "true");
+
+ assertSame(compiled, handler._result);
+ assertEquals(compiled.getOperation(), DdlOperation.CREATE_TABLE);
+ }
+
+ @Test
+ public void unqualifiedNameYieldsNullDatabase() {
+ RecordingHandler handler = new RecordingHandler();
+ DdlCompiler.setCreateTableWithOptionsHandler(handler);
+ DdlCompiler.compile("CREATE TABLE trips WITH (type = 'iceberg')",
DdlCompileContext.STATELESS);
+ assertNull(handler._databaseName);
+ assertEquals(handler._tableName, "trips");
+ assertFalse(handler._ifNotExists);
+ }
+
+ @Test
+ public void emptyOptionsRejectedBeforeHandler() {
+ RecordingHandler handler = new RecordingHandler();
+ DdlCompiler.setCreateTableWithOptionsHandler(handler);
+ DdlCompilationException e = expectThrows(DdlCompilationException.class,
+ () -> DdlCompiler.compile("CREATE TABLE trips WITH ()",
DdlCompileContext.STATELESS));
+ assertTrue(e.getMessage().contains("at least one option"),
+ "Expected empty-options rejection in: " + e.getMessage());
+ assertNull(handler._tableName, "Handler must not be invoked for empty
options");
+ }
+
+ @Test
+ public void duplicateOptionKeysRejectedBeforeHandler() {
+ RecordingHandler handler = new RecordingHandler();
+ DdlCompiler.setCreateTableWithOptionsHandler(handler);
+ // Duplicate detection is case-insensitive, matching PROPERTIES behavior.
+ DdlCompilationException e = expectThrows(DdlCompilationException.class,
+ () -> DdlCompiler.compile("CREATE TABLE trips WITH (type = 'iceberg',
'TYPE' = 'delta')",
+ DdlCompileContext.STATELESS));
+ assertTrue(e.getMessage().contains("Duplicate property key"),
+ "Expected duplicate-key rejection in: " + e.getMessage());
+ assertNull(handler._tableName, "Handler must not be invoked for duplicate
options");
+ }
+
+ @Test
+ public void handlerCannotRedirectStatementIdentity() {
+ // The statement identity is fixed by the SQL text — authorization and
persistence act on the
+ // names carried by the returned object, so the compiler must reject a
handler that returns a
+ // different database/table/schema name or IF NOT EXISTS flag.
+ DdlCompiler.setCreateTableWithOptionsHandler(
+ (databaseName, tableName, ifNotExists, options, ctx) -> compiledFor(
+ databaseName, "hijacked", "hijacked", ifNotExists));
+ assertIdentityRejected("table name");
+
+ DdlCompiler.setCreateTableWithOptionsHandler(
+ (databaseName, tableName, ifNotExists, options, ctx) -> compiledFor(
+ "otherDb", tableName, tableName, ifNotExists));
+ assertIdentityRejected("database name");
+
+ DdlCompiler.setCreateTableWithOptionsHandler(
+ (databaseName, tableName, ifNotExists, options, ctx) -> compiledFor(
+ databaseName, tableName, "otherSchema", ifNotExists));
+ assertIdentityRejected("schema named");
+
+ DdlCompiler.setCreateTableWithOptionsHandler(
+ (databaseName, tableName, ifNotExists, options, ctx) -> compiledFor(
+ databaseName, tableName, tableName, !ifNotExists));
+ assertIdentityRejected("IF NOT EXISTS");
+
+ DdlCompiler.setCreateTableWithOptionsHandler(
+ (databaseName, tableName, ifNotExists, options, ctx) -> new
CompiledCreateTable(
+ databaseName, null, new
TableConfigBuilder(TableType.OFFLINE).setTableName(tableName).build(),
+ ifNotExists, new ArrayList<>()));
+ assertIdentityRejected("schema");
+ }
+
+ private static CompiledCreateTable compiledFor(@Nullable String
databaseName, String tableName,
+ String schemaName, boolean ifNotExists) {
+ Schema schema = new Schema();
+ schema.setSchemaName(schemaName);
+ TableConfig tableConfig = new
TableConfigBuilder(TableType.OFFLINE).setTableName(tableName).build();
+ return new CompiledCreateTable(databaseName, schema, tableConfig,
ifNotExists, new ArrayList<>());
+ }
+
+ private static void assertIdentityRejected(String expectedFragment) {
+ DdlCompilationException e = expectThrows(DdlCompilationException.class,
+ () -> DdlCompiler.compile("CREATE TABLE trips WITH (type = 'iceberg')",
+ DdlCompileContext.STATELESS));
+ assertTrue(e.getMessage().contains(expectedFragment),
+ "Expected '" + expectedFragment + "' in: " + e.getMessage());
+ }
+
+ @Test
+ public void nullHandlerResultRejected() {
+ DdlCompiler.setCreateTableWithOptionsHandler(
+ (databaseName, tableName, ifNotExists, options, ctx) -> null);
+ DdlCompilationException e = expectThrows(DdlCompilationException.class,
+ () -> DdlCompiler.compile("CREATE TABLE trips WITH (type =
'iceberg')", DdlCompileContext.STATELESS));
+ assertTrue(e.getMessage().contains("returned no result"),
+ "Expected null-result rejection in: " + e.getMessage());
+ }
+
+ @Test
+ public void columnListFormBypassesHandler() {
+ RecordingHandler handler = new RecordingHandler();
+ DdlCompiler.setCreateTableWithOptionsHandler(handler);
+ CompiledDdl compiled = DdlCompiler.compile(
+ "CREATE TABLE events (id INT, name STRING) TABLE_TYPE = OFFLINE",
DdlCompileContext.STATELESS);
+ assertEquals(compiled.getOperation(), DdlOperation.CREATE_TABLE);
+ assertNull(handler._tableName, "Column-list form must not route through
the options handler");
+ }
+
+ /// Records the arguments it was called with and returns a minimal valid
compilation result.
+ private static final class RecordingHandler implements
CreateTableWithOptionsHandler {
+ @Nullable
+ private String _databaseName;
+ @Nullable
+ private String _tableName;
+ private boolean _ifNotExists;
+ private Map<String, String> _options = new LinkedHashMap<>();
+ @Nullable
+ private CompiledCreateTable _result;
+
+ @Override
+ public CompiledCreateTable compile(@Nullable String databaseName, String
tableName, boolean ifNotExists,
+ Map<String, String> options, DdlCompileContext ctx) {
+ _databaseName = databaseName;
+ _tableName = tableName;
+ _ifNotExists = ifNotExists;
+ _options = options;
+ Schema schema = new Schema();
+ schema.setSchemaName(tableName);
+ TableConfig tableConfig = new
TableConfigBuilder(TableType.OFFLINE).setTableName(tableName).build();
+ _result = new CompiledCreateTable(databaseName, schema, tableConfig,
ifNotExists, new ArrayList<>());
+ return _result;
+ }
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
