Copilot commented on code in PR #1847:
URL: https://github.com/apache/fluss/pull/1847#discussion_r2514016487
##########
fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/LakeEnabledTableCreateITCase.java:
##########
@@ -610,6 +809,84 @@ void testAlterLakeEnabledTableProperties() throws
Exception {
admin.alterTable(TablePath.of(DATABASE, "not_exist_table"),
tableChanges, true).get();
}
+ @Test
+ void testEnableLakeTableAfterAlterTableProperties() throws Exception {
+ Map<String, String> customProperties = new HashMap<>();
+ customProperties.put("k1", "v1");
+ customProperties.put("paimon.file.format", "parquet");
+
+ // create table
+ TableDescriptor tableDescriptor =
+ TableDescriptor.builder()
+ .schema(
+ Schema.newBuilder()
+ .column("c1", DataTypes.INT())
+ .column("c2", DataTypes.STRING())
+ .build())
+ .property(ConfigOptions.TABLE_DATALAKE_ENABLED, false)
+ .customProperties(customProperties)
+ .distributedBy(BUCKET_NUM, "c1", "c2")
+ .build();
+ TablePath tablePath = TablePath.of(DATABASE,
"enable_lake_table_after_alter_properties");
+ admin.createTable(tablePath, tableDescriptor, false).get();
+ // paimon table should not exist because lake table is disable
+ assertThatThrownBy(
+ () ->
+ paimonCatalog.getTable(
+ Identifier.create(DATABASE,
tablePath.getTableName())))
+ .isInstanceOf(Catalog.TableNotExistException.class)
+ .hasMessage(String.format("Table %s does not exist.",
tablePath));
+
+ // alter fluss tale properties
+ List<TableChange> tableChanges =
+ Arrays.asList(TableChange.reset("k1"), TableChange.set("k2",
"v2"));
+ admin.alterTable(tablePath, tableChanges, false).get();
+ // enable lake table should be ok
+ TableChange.SetOption enableLake =
+ TableChange.set(ConfigOptions.TABLE_DATALAKE_ENABLED.key(),
"true");
+ admin.alterTable(tablePath, Collections.singletonList(enableLake),
false).get();
+ Table paimonTable =
+ paimonCatalog.getTable(Identifier.create(DATABASE,
tablePath.getTableName()));
+ customProperties.remove("k1");
+ customProperties.put("k2", "v2");
+ Map<String, String> newProperties = new
HashMap<>(tableDescriptor.getProperties());
+ newProperties.put(ConfigOptions.TABLE_DATALAKE_ENABLED.key(), "true");
+ tableDescriptor = tableDescriptor.withProperties(newProperties,
customProperties);
+ verifyPaimonTable(
+ paimonTable,
+ tableDescriptor,
+ RowType.of(
+ new DataType[] {
+ org.apache.paimon.types.DataTypes.INT(),
+ org.apache.paimon.types.DataTypes.STRING(),
+ // for __bucket, __offset, __timestamp
+ org.apache.paimon.types.DataTypes.INT(),
+ org.apache.paimon.types.DataTypes.BIGINT(),
+
org.apache.paimon.types.DataTypes.TIMESTAMP_WITH_LOCAL_TIME_ZONE()
+ },
+ new String[] {
+ "c1",
+ "c2",
+ BUCKET_COLUMN_NAME,
+ OFFSET_COLUMN_NAME,
+ TIMESTAMP_COLUMN_NAME
+ }),
+ "c1,c2",
+ BUCKET_NUM);
+
+ // disable lake table
+ TableChange.SetOption diableLake =
+ TableChange.set(ConfigOptions.TABLE_DATALAKE_ENABLED.key(),
"false");
+ admin.alterTable(tablePath, Collections.singletonList(diableLake),
false).get();
Review Comment:
Corrected spelling of 'diableLake' to 'disableLake'.
```suggestion
TableChange.SetOption disableLake =
TableChange.set(ConfigOptions.TABLE_DATALAKE_ENABLED.key(),
"false");
admin.alterTable(tablePath, Collections.singletonList(disableLake),
false).get();
```
##########
fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/LakeEnabledTableCreateITCase.java:
##########
@@ -610,6 +809,84 @@ void testAlterLakeEnabledTableProperties() throws
Exception {
admin.alterTable(TablePath.of(DATABASE, "not_exist_table"),
tableChanges, true).get();
}
+ @Test
+ void testEnableLakeTableAfterAlterTableProperties() throws Exception {
+ Map<String, String> customProperties = new HashMap<>();
+ customProperties.put("k1", "v1");
+ customProperties.put("paimon.file.format", "parquet");
+
+ // create table
+ TableDescriptor tableDescriptor =
+ TableDescriptor.builder()
+ .schema(
+ Schema.newBuilder()
+ .column("c1", DataTypes.INT())
+ .column("c2", DataTypes.STRING())
+ .build())
+ .property(ConfigOptions.TABLE_DATALAKE_ENABLED, false)
+ .customProperties(customProperties)
+ .distributedBy(BUCKET_NUM, "c1", "c2")
+ .build();
+ TablePath tablePath = TablePath.of(DATABASE,
"enable_lake_table_after_alter_properties");
+ admin.createTable(tablePath, tableDescriptor, false).get();
+ // paimon table should not exist because lake table is disable
+ assertThatThrownBy(
+ () ->
+ paimonCatalog.getTable(
+ Identifier.create(DATABASE,
tablePath.getTableName())))
+ .isInstanceOf(Catalog.TableNotExistException.class)
+ .hasMessage(String.format("Table %s does not exist.",
tablePath));
+
+ // alter fluss tale properties
Review Comment:
Corrected spelling of 'tale' to 'table' in comment.
##########
fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/utils/PaimonSchemaValidation.java:
##########
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.fluss.lake.paimon.utils;
+
+import org.apache.fluss.config.ConfigOptions;
+import org.apache.fluss.exception.TableAlreadyExistException;
+
+import org.apache.paimon.CoreOptions;
+import org.apache.paimon.schema.Schema;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import static
org.apache.fluss.lake.paimon.utils.PaimonConversions.FLUSS_CONF_PREFIX;
+
+/** Validator of Paimon table schema. */
+public class PaimonSchemaValidation {
+
+ public static void validatePaimonSchemaCapability(Schema existingSchema,
Schema newSchema) {
+ // check fields
+ if (!existingSchema.fields().equals(newSchema.fields())) {
+ throw new TableAlreadyExistException(
+ String.format(
+ "The fields of the existing Paimon table are not
compatible with those of the new table to be created. "
+ + "Existing fields: %s, new fields: %s.",
+ existingSchema.fields(), newSchema.fields()));
+ }
+
+ // check pks
+ if (!existingSchema.primaryKeys().equals(newSchema.primaryKeys())) {
+ throw new TableAlreadyExistException(
+ String.format(
+ "The primary keys of the existing Paimon table are
not compatible with those of the new table to be created. "
+ + "Existing primary keys: %s, new primary
keys: %s.",
+ existingSchema.primaryKeys(),
newSchema.primaryKeys()));
+ }
+
+ // check partition keys
+ if (!existingSchema.partitionKeys().equals(newSchema.partitionKeys()))
{
+ throw new TableAlreadyExistException(
+ String.format(
+ "The partition keys of the existing Paimon table
are not compatible with those of the new table to be created. "
+ + "Existing partition keys: %s, new
partition keys: %s.",
+ existingSchema.partitionKeys(),
newSchema.partitionKeys()));
+ }
+
+ // check options
+ Map<String, String> existingOptions = new
HashMap<>(existingSchema.options());
+ Map<String, String> newOptions = new HashMap<>(newSchema.options());
+ // `path` will be set automatically by Paimon, so we need to remove it
here
+ existingOptions.remove(CoreOptions.PATH.key());
+ // when enable datalake with an existing table, we should ignore the
+ // `table.datalake.enabled`
+ String datalakeConfigKey = FLUSS_CONF_PREFIX +
ConfigOptions.TABLE_DATALAKE_ENABLED.key();
+ if
(Boolean.FALSE.toString().equals(existingOptions.get(datalakeConfigKey))) {
+ existingOptions.remove(datalakeConfigKey);
+ newOptions.remove(datalakeConfigKey);
+ }
+ if (!existingOptions.equals(newOptions)) {
+ throw new TableAlreadyExistException(
+ String.format(
+ "The options of the existing Paimon table are not
compatible with those of the new table to be created. "
+ + "Existing options: %s, new options: %s.",
+ existingOptions, newSchema.options()));
Review Comment:
The error message is inconsistent with the comparison being performed. The
code compares `existingOptions` (modified copy) with `newOptions` (modified
copy) at line 74, but the error message at line 79 uses `newSchema.options()`
(original unmodified options) instead of `newOptions`. This will display
incorrect options in the error message. Change `newSchema.options()` to
`newOptions`.
```suggestion
existingOptions, newOptions));
```
##########
fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/LakeEnabledTableCreateITCase.java:
##########
@@ -610,6 +809,84 @@ void testAlterLakeEnabledTableProperties() throws
Exception {
admin.alterTable(TablePath.of(DATABASE, "not_exist_table"),
tableChanges, true).get();
}
+ @Test
+ void testEnableLakeTableAfterAlterTableProperties() throws Exception {
+ Map<String, String> customProperties = new HashMap<>();
+ customProperties.put("k1", "v1");
+ customProperties.put("paimon.file.format", "parquet");
+
+ // create table
+ TableDescriptor tableDescriptor =
+ TableDescriptor.builder()
+ .schema(
+ Schema.newBuilder()
+ .column("c1", DataTypes.INT())
+ .column("c2", DataTypes.STRING())
+ .build())
+ .property(ConfigOptions.TABLE_DATALAKE_ENABLED, false)
+ .customProperties(customProperties)
+ .distributedBy(BUCKET_NUM, "c1", "c2")
+ .build();
+ TablePath tablePath = TablePath.of(DATABASE,
"enable_lake_table_after_alter_properties");
+ admin.createTable(tablePath, tableDescriptor, false).get();
+ // paimon table should not exist because lake table is disable
+ assertThatThrownBy(
+ () ->
+ paimonCatalog.getTable(
+ Identifier.create(DATABASE,
tablePath.getTableName())))
+ .isInstanceOf(Catalog.TableNotExistException.class)
+ .hasMessage(String.format("Table %s does not exist.",
tablePath));
+
+ // alter fluss tale properties
+ List<TableChange> tableChanges =
+ Arrays.asList(TableChange.reset("k1"), TableChange.set("k2",
"v2"));
+ admin.alterTable(tablePath, tableChanges, false).get();
+ // enable lake table should be ok
+ TableChange.SetOption enableLake =
+ TableChange.set(ConfigOptions.TABLE_DATALAKE_ENABLED.key(),
"true");
+ admin.alterTable(tablePath, Collections.singletonList(enableLake),
false).get();
+ Table paimonTable =
+ paimonCatalog.getTable(Identifier.create(DATABASE,
tablePath.getTableName()));
+ customProperties.remove("k1");
+ customProperties.put("k2", "v2");
+ Map<String, String> newProperties = new
HashMap<>(tableDescriptor.getProperties());
+ newProperties.put(ConfigOptions.TABLE_DATALAKE_ENABLED.key(), "true");
+ tableDescriptor = tableDescriptor.withProperties(newProperties,
customProperties);
+ verifyPaimonTable(
+ paimonTable,
+ tableDescriptor,
+ RowType.of(
+ new DataType[] {
+ org.apache.paimon.types.DataTypes.INT(),
+ org.apache.paimon.types.DataTypes.STRING(),
+ // for __bucket, __offset, __timestamp
+ org.apache.paimon.types.DataTypes.INT(),
+ org.apache.paimon.types.DataTypes.BIGINT(),
+
org.apache.paimon.types.DataTypes.TIMESTAMP_WITH_LOCAL_TIME_ZONE()
+ },
+ new String[] {
+ "c1",
+ "c2",
+ BUCKET_COLUMN_NAME,
+ OFFSET_COLUMN_NAME,
+ TIMESTAMP_COLUMN_NAME
+ }),
+ "c1,c2",
+ BUCKET_NUM);
+
+ // disable lake table
+ TableChange.SetOption diableLake =
+ TableChange.set(ConfigOptions.TABLE_DATALAKE_ENABLED.key(),
"false");
+ admin.alterTable(tablePath, Collections.singletonList(diableLake),
false).get();
+
+ // alter fluss tale properties when lake table is disabled
Review Comment:
Corrected spelling of 'tale' to 'table' in comment.
```suggestion
// alter fluss table properties when lake table is disabled
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
