JingsongLi commented on code in PR #10833:
URL: https://github.com/apache/paimon/pull/10833#discussion_r4456882011


##########
paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lakestream/LakeStreamTableFactory.java:
##########
@@ -0,0 +1,336 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.flink.lakestream;
+
+import org.apache.paimon.CoreOptions;
+import org.apache.paimon.catalog.Catalog;
+import org.apache.paimon.catalog.CatalogContext;
+import org.apache.paimon.catalog.Identifier;
+import org.apache.paimon.flink.AbstractFlinkTableFactory;
+import org.apache.paimon.flink.FlinkCatalog;
+import org.apache.paimon.flink.FlinkCatalogFactory;
+import org.apache.paimon.flink.FlinkFileIOLoader;
+import org.apache.paimon.options.CatalogOptions;
+import org.apache.paimon.options.Options;
+import org.apache.paimon.table.FileStoreTableFactory;
+import org.apache.paimon.table.FormatTable;
+import org.apache.paimon.table.Table;
+
+import org.apache.flink.configuration.ConfigOption;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.configuration.ReadableConfig;
+import org.apache.flink.table.api.ValidationException;
+import org.apache.flink.table.catalog.CatalogBaseTable;
+import org.apache.flink.table.catalog.CatalogTable;
+import org.apache.flink.table.catalog.ObjectIdentifier;
+import org.apache.flink.table.catalog.ResolvedCatalogTable;
+import org.apache.flink.table.connector.sink.DynamicTableSink;
+import org.apache.flink.table.connector.source.DynamicTableSource;
+import org.apache.flink.table.factories.DynamicTableFactory;
+import org.apache.flink.table.factories.DynamicTableSinkFactory;
+import org.apache.flink.table.factories.DynamicTableSourceFactory;
+import org.apache.flink.table.factories.Factory;
+
+import javax.annotation.Nullable;
+
+import java.util.Collections;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+
+/**
+ * Builds the source and the sink of a lake-stream table by delegating to the 
Lakestream Store's own
+ * table factory. Not an SPI-registered factory: {@link #factoryIdentifier} is 
implemented only
+ * because the interface requires it.
+ */
+public class LakeStreamTableFactory implements DynamicTableSourceFactory, 
DynamicTableSinkFactory {
+
+    private final FlinkCatalog catalog;
+
+    public LakeStreamTableFactory(FlinkCatalog catalog) {
+        this.catalog = catalog;
+    }
+
+    @Override
+    public String factoryIdentifier() {
+        return FlinkCatalogFactory.IDENTIFIER;
+    }
+
+    @Override
+    public Set<ConfigOption<?>> requiredOptions() {
+        return Collections.emptySet();
+    }
+
+    @Override
+    public Set<ConfigOption<?>> optionalOptions() {
+        return Collections.emptySet();
+    }
+
+    /** Whether the read asks for the Lakestream Store's view of a table. */
+    public static boolean isLakeStreamRead(
+            @Nullable FlinkCatalog catalog, DynamicTableFactory.Context 
context) {
+        if (catalog == null) {
+            // with no catalog there is no lake-stream table to confirm the 
name against, so the
+            // name alone counts; the read is then refused for lack of a store 
rather than served
+            // from Paimon
+            return 
LakeStreamUtils.isRealTimeTable(context.getObjectIdentifier().toObjectPath());
+        }
+        return realTimeReadBaseTable(catalog.catalog(), context) != null;
+    }
+
+    /**
+     * Whether the write is to a lake-stream table, or through the read-only 
{@code $rt} name.
+     * Connector options and query hints do not change the answer.
+     */
+    public static boolean isLakeStreamWrite(
+            @Nullable FlinkCatalog catalog, DynamicTableFactory.Context 
context) {
+        Identifier identifier = identifierOf(context);
+        if (LakeStreamUtils.isRealTimeTable(identifier)) {
+            return true;
+        }
+        return catalog != null
+                ? persistedLakeStreamTable(catalog.catalog(), identifier) != 
null
+                : lakeStreamEnabledAtPath(context);
+    }
+
+    @Override
+    public DynamicTableSource 
createDynamicTableSource(DynamicTableFactory.Context context) {
+        Table baseTable = realTimeReadBaseTable(catalog.catalog(), context);
+        ObjectIdentifier baseIdentifier =
+                
LakeStreamUtils.stripRealTimeSuffix(context.getObjectIdentifier());
+        checkNotFormatTable(baseTable, baseIdentifier);
+        org.apache.flink.table.catalog.Catalog storeCatalog =
+                catalog.getOrCreateLakeStreamStoreCatalog();
+        // the catalog already resolved this read against the store, so the 
planner's table is the
+        // store's table with any query hints applied on top of it
+        CatalogBaseTable storeTable = context.getCatalogTable();
+        DynamicTableSourceFactory sourceFactory =
+                storeFactory(
+                        storeCatalog,
+                        DynamicTableSourceFactory.class,
+                        "create a source for '$rt'-suffixed reads");
+        return sourceFactory.createDynamicTableSource(
+                delegatedContext(context, baseIdentifier, storeTable));
+    }
+
+    @Override
+    public DynamicTableSink createDynamicTableSink(DynamicTableFactory.Context 
context) {
+        checkNotRealTimeWrite(context);
+        Table table = persistedLakeStreamTable(catalog.catalog(), 
identifierOf(context));
+        checkNotFormatTable(table, context.getObjectIdentifier());
+        ObjectIdentifier identifier = context.getObjectIdentifier();
+        org.apache.flink.table.catalog.Catalog storeCatalog =
+                catalog.getOrCreateLakeStreamStoreCatalog();
+        // the store table is fetched from the store catalog because the 
store's sink factory may
+        // require provider-specific table options (PIP-39 L289)
+        CatalogBaseTable storeTable =
+                catalog.requireLakeStreamStoreTable(identifier.toObjectPath());
+        DynamicTableSinkFactory sinkFactory =
+                storeFactory(
+                        storeCatalog, DynamicTableSinkFactory.class, "create a 
sink for writes");
+        return new LakeStreamDynamicTableSink(
+                sinkFactory.createDynamicTableSink(
+                        delegatedContext(context, identifier, storeTable)),
+                identifier);
+    }
+
+    /** A format table has no Paimon history for the store to tier into, so it 
cannot be one. */
+    private void checkNotFormatTable(@Nullable Table table, ObjectIdentifier 
identifier) {
+        if (table instanceof FormatTable) {
+            throw new ValidationException(
+                    String.format(
+                            "Table '%s' cannot be a lake-stream table: '%s' is 
only supported on "
+                                    + "Paimon tables, not on format tables.",
+                            identifier.asSummaryString(), 
CoreOptions.LAKE_STREAM_ENABLED.key()));
+        }
+    }
+
+    /** The base table of a {@code $rt} read, or {@code null} when the read 
stays on Paimon. */
+    @Nullable
+    private static Table realTimeReadBaseTable(
+            Catalog paimonCatalog, DynamicTableFactory.Context context) {
+        Identifier identifier = identifierOf(context);
+        if (!LakeStreamUtils.isRealTimeTable(identifier)) {
+            return null;
+        }
+        return persistedLakeStreamTable(
+                paimonCatalog,
+                Identifier.create(identifier.getDatabaseName(), 
identifier.getTableName()));
+    }
+
+    /** The Paimon table when lake-stream mode is enabled on it, otherwise 
{@code null}. */
+    @Nullable
+    private static Table persistedLakeStreamTable(Catalog paimonCatalog, 
Identifier identifier) {
+        try {
+            Table table = paimonCatalog.getTable(identifier);
+            return 
Options.fromMap(table.options()).get(CoreOptions.LAKE_STREAM_ENABLED)
+                    ? table
+                    : null;
+        } catch (Catalog.TableNotExistException e) {
+            return null;
+        }
+    }
+
+    /** Whether the schema persisted at the connector's {@code path} has 
lake-stream mode on. */
+    private static boolean lakeStreamEnabledAtPath(DynamicTableFactory.Context 
context) {
+        Options options = 
Options.fromMap(context.getCatalogTable().getOptions());
+        String path = options.get(CoreOptions.PATH);
+        if (path == null) {
+            return options.get(CoreOptions.LAKE_STREAM_ENABLED);
+        }
+        // only the location goes in: the factory merges whatever it is given 
over the persisted
+        // schema, so passing the connector options would restate the very 
flag being read
+        Options location = new Options();
+        location.set(CoreOptions.PATH, path);
+        location.set(CoreOptions.BRANCH, options.get(CoreOptions.BRANCH));
+        Table table =
+                FileStoreTableFactory.create(
+                        CatalogContext.create(location, new 
FlinkFileIOLoader()));
+        return 
Options.fromMap(table.options()).get(CoreOptions.LAKE_STREAM_ENABLED);
+    }
+
+    private static Identifier identifierOf(DynamicTableFactory.Context 
context) {
+        ObjectIdentifier identifier = context.getObjectIdentifier();
+        return new Identifier(identifier.getDatabaseName(), 
identifier.getObjectName());
+    }
+
+    /**
+     * The failure reported for a lake-stream table when no store was 
configured. Serving it from
+     * Paimon instead would leave the store and Paimon holding different 
values for the same keys.
+     *
+     * @param operation what the statement was trying to do, e.g. {@code 
"write to"}
+     */
+    public static ValidationException missingStoreError(
+            ObjectIdentifier identifier, String operation) {
+        return new ValidationException(
+                String.format(
+                        "Cannot %s lake-stream table '%s': the catalog was 
created without "
+                                + "'%s', so there is no Lakestream Store to 
serve it, and serving it "
+                                + "from Paimon instead would leave the store 
and Paimon holding "
+                                + "different data for the same keys. Create 
the catalog with '%s' and "
+                                + "the store's connection options, or set '%s' 
to false first.",
+                        operation,
+                        identifier.asSummaryString(),
+                        CatalogOptions.LAKE_STREAM_STORE.key(),
+                        CatalogOptions.LAKE_STREAM_STORE.key(),
+                        CoreOptions.LAKE_STREAM_ENABLED.key()));
+    }
+
+    /** Rejects a write through the read-only {@code $rt} name, which the 
store serves. */
+    private static void checkNotRealTimeWrite(DynamicTableFactory.Context 
context) {
+        ObjectIdentifier identifier = context.getObjectIdentifier();
+        if (LakeStreamUtils.isRealTimeTable(identifierOf(context))) {
+            throw new ValidationException(
+                    String.format(
+                            "Cannot write to '%s': the '$%s' suffix names the 
Lakestream "
+                                    + "Store's read-only view of a lake-stream 
table. Write to "
+                                    + "'%s' instead.",
+                            identifier.asSummaryString(),
+                            LakeStreamUtils.REAL_TIME_TABLE_SUFFIX,
+                            
LakeStreamUtils.stripRealTimeSuffix(identifier).asSummaryString()));
+        }
+    }
+
+    /** The store's factory of the given type, failing with an error naming 
the store. */
+    private <T> T storeFactory(
+            org.apache.flink.table.catalog.Catalog storeCatalog,
+            Class<T> factoryClass,
+            String purpose) {
+        String store = catalog.lakeStreamStoreIdentifier();
+        Optional<Factory> factory = storeCatalog.getFactory();
+        if (!factory.isPresent() || !factoryClass.isInstance(factory.get())) {
+            throw new ValidationException(
+                    String.format(
+                            "The Lakestream Store '%s' (configured via '%s' = 
'%s') does not "
+                                    + "provide a %s through its catalog 
factory, so Paimon cannot "
+                                    + "delegate lake-stream operations to it 
and cannot %s.",
+                            store,
+                            CatalogOptions.LAKE_STREAM_STORE.key(),
+                            store,
+                            factoryClass.getSimpleName(),
+                            purpose));
+        }
+        return factoryClass.cast(factory.get());
+    }
+
+    /**
+     * The planner's context adapted for the store's factory: the base 
identifier, the store table's
+     * options, and the store's catalog options after prefix removal.
+     */
+    private DynamicTableFactory.Context delegatedContext(
+            DynamicTableFactory.Context original,
+            ObjectIdentifier identifier,
+            CatalogBaseTable storeTable) {
+        if (!(storeTable instanceof CatalogTable)) {
+            throw new ValidationException(
+                    String.format(
+                            "Expected a table from the Lakestream Store '%s' 
for '%s', but got %s.",
+                            catalog.lakeStreamStoreIdentifier(),
+                            identifier.asSummaryString(),
+                            storeTable.getClass().getName()));
+        }
+        ResolvedCatalogTable resolvedStoreTable =
+                new ResolvedCatalogTable(
+                        (CatalogTable) storeTable, 
original.getCatalogTable().getResolvedSchema());

Review Comment:
   [P1] Validate the Paimon/store row layout before delegating writes
   
   The sink context combines the store table with Paimon's current resolved 
schema without checking their column order. The documentation and new catalog 
test explicitly allow Paimon-only column changes while the store catches up. 
With real Fluss 1.0, I created both tables as `(a INT,b INT)`, inserted 
`(1,2)`, ran `ALTER TABLE orders MODIFY b INT FIRST` only in Paimon, and 
inserted `(4,3)` (therefore b=4,a=3). `SELECT a,b FROM orders$rt` returned 
`(1,2),(4,3)` instead of `(1,2),(3,4)`: the second row is silently stored under 
the wrong columns. Fluss's serializer maps names only when the column counts 
differ; equal-width rows are positional. Reject incompatible physical 
schemas/order before delegating, or explicitly align the emitted rows with the 
store schema, and cover the supported schema-drift interval with a real-store 
test.



##########
paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lakestream/LakeStreamTableFactory.java:
##########
@@ -0,0 +1,336 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.flink.lakestream;
+
+import org.apache.paimon.CoreOptions;
+import org.apache.paimon.catalog.Catalog;
+import org.apache.paimon.catalog.CatalogContext;
+import org.apache.paimon.catalog.Identifier;
+import org.apache.paimon.flink.AbstractFlinkTableFactory;
+import org.apache.paimon.flink.FlinkCatalog;
+import org.apache.paimon.flink.FlinkCatalogFactory;
+import org.apache.paimon.flink.FlinkFileIOLoader;
+import org.apache.paimon.options.CatalogOptions;
+import org.apache.paimon.options.Options;
+import org.apache.paimon.table.FileStoreTableFactory;
+import org.apache.paimon.table.FormatTable;
+import org.apache.paimon.table.Table;
+
+import org.apache.flink.configuration.ConfigOption;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.configuration.ReadableConfig;
+import org.apache.flink.table.api.ValidationException;
+import org.apache.flink.table.catalog.CatalogBaseTable;
+import org.apache.flink.table.catalog.CatalogTable;
+import org.apache.flink.table.catalog.ObjectIdentifier;
+import org.apache.flink.table.catalog.ResolvedCatalogTable;
+import org.apache.flink.table.connector.sink.DynamicTableSink;
+import org.apache.flink.table.connector.source.DynamicTableSource;
+import org.apache.flink.table.factories.DynamicTableFactory;
+import org.apache.flink.table.factories.DynamicTableSinkFactory;
+import org.apache.flink.table.factories.DynamicTableSourceFactory;
+import org.apache.flink.table.factories.Factory;
+
+import javax.annotation.Nullable;
+
+import java.util.Collections;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+
+/**
+ * Builds the source and the sink of a lake-stream table by delegating to the 
Lakestream Store's own
+ * table factory. Not an SPI-registered factory: {@link #factoryIdentifier} is 
implemented only
+ * because the interface requires it.
+ */
+public class LakeStreamTableFactory implements DynamicTableSourceFactory, 
DynamicTableSinkFactory {
+
+    private final FlinkCatalog catalog;
+
+    public LakeStreamTableFactory(FlinkCatalog catalog) {
+        this.catalog = catalog;
+    }
+
+    @Override
+    public String factoryIdentifier() {
+        return FlinkCatalogFactory.IDENTIFIER;
+    }
+
+    @Override
+    public Set<ConfigOption<?>> requiredOptions() {
+        return Collections.emptySet();
+    }
+
+    @Override
+    public Set<ConfigOption<?>> optionalOptions() {
+        return Collections.emptySet();
+    }
+
+    /** Whether the read asks for the Lakestream Store's view of a table. */
+    public static boolean isLakeStreamRead(
+            @Nullable FlinkCatalog catalog, DynamicTableFactory.Context 
context) {
+        if (catalog == null) {
+            // with no catalog there is no lake-stream table to confirm the 
name against, so the
+            // name alone counts; the read is then refused for lack of a store 
rather than served
+            // from Paimon
+            return 
LakeStreamUtils.isRealTimeTable(context.getObjectIdentifier().toObjectPath());
+        }
+        return realTimeReadBaseTable(catalog.catalog(), context) != null;
+    }
+
+    /**
+     * Whether the write is to a lake-stream table, or through the read-only 
{@code $rt} name.
+     * Connector options and query hints do not change the answer.
+     */
+    public static boolean isLakeStreamWrite(
+            @Nullable FlinkCatalog catalog, DynamicTableFactory.Context 
context) {
+        Identifier identifier = identifierOf(context);
+        if (LakeStreamUtils.isRealTimeTable(identifier)) {
+            return true;
+        }
+        return catalog != null
+                ? persistedLakeStreamTable(catalog.catalog(), identifier) != 
null
+                : lakeStreamEnabledAtPath(context);
+    }
+
+    @Override
+    public DynamicTableSource 
createDynamicTableSource(DynamicTableFactory.Context context) {
+        Table baseTable = realTimeReadBaseTable(catalog.catalog(), context);
+        ObjectIdentifier baseIdentifier =
+                
LakeStreamUtils.stripRealTimeSuffix(context.getObjectIdentifier());
+        checkNotFormatTable(baseTable, baseIdentifier);
+        org.apache.flink.table.catalog.Catalog storeCatalog =
+                catalog.getOrCreateLakeStreamStoreCatalog();
+        // the catalog already resolved this read against the store, so the 
planner's table is the
+        // store's table with any query hints applied on top of it
+        CatalogBaseTable storeTable = context.getCatalogTable();
+        DynamicTableSourceFactory sourceFactory =
+                storeFactory(
+                        storeCatalog,
+                        DynamicTableSourceFactory.class,
+                        "create a source for '$rt'-suffixed reads");
+        return sourceFactory.createDynamicTableSource(
+                delegatedContext(context, baseIdentifier, storeTable));
+    }
+
+    @Override
+    public DynamicTableSink createDynamicTableSink(DynamicTableFactory.Context 
context) {
+        checkNotRealTimeWrite(context);
+        Table table = persistedLakeStreamTable(catalog.catalog(), 
identifierOf(context));
+        checkNotFormatTable(table, context.getObjectIdentifier());
+        ObjectIdentifier identifier = context.getObjectIdentifier();
+        org.apache.flink.table.catalog.Catalog storeCatalog =
+                catalog.getOrCreateLakeStreamStoreCatalog();
+        // the store table is fetched from the store catalog because the 
store's sink factory may
+        // require provider-specific table options (PIP-39 L289)
+        CatalogBaseTable storeTable =
+                catalog.requireLakeStreamStoreTable(identifier.toObjectPath());
+        DynamicTableSinkFactory sinkFactory =
+                storeFactory(
+                        storeCatalog, DynamicTableSinkFactory.class, "create a 
sink for writes");
+        return new LakeStreamDynamicTableSink(
+                sinkFactory.createDynamicTableSink(
+                        delegatedContext(context, identifier, storeTable)),
+                identifier);
+    }
+
+    /** A format table has no Paimon history for the store to tier into, so it 
cannot be one. */
+    private void checkNotFormatTable(@Nullable Table table, ObjectIdentifier 
identifier) {
+        if (table instanceof FormatTable) {
+            throw new ValidationException(
+                    String.format(
+                            "Table '%s' cannot be a lake-stream table: '%s' is 
only supported on "
+                                    + "Paimon tables, not on format tables.",
+                            identifier.asSummaryString(), 
CoreOptions.LAKE_STREAM_ENABLED.key()));
+        }
+    }
+
+    /** The base table of a {@code $rt} read, or {@code null} when the read 
stays on Paimon. */
+    @Nullable
+    private static Table realTimeReadBaseTable(
+            Catalog paimonCatalog, DynamicTableFactory.Context context) {
+        Identifier identifier = identifierOf(context);
+        if (!LakeStreamUtils.isRealTimeTable(identifier)) {
+            return null;
+        }
+        return persistedLakeStreamTable(
+                paimonCatalog,
+                Identifier.create(identifier.getDatabaseName(), 
identifier.getTableName()));
+    }
+
+    /** The Paimon table when lake-stream mode is enabled on it, otherwise 
{@code null}. */
+    @Nullable
+    private static Table persistedLakeStreamTable(Catalog paimonCatalog, 
Identifier identifier) {
+        try {
+            Table table = paimonCatalog.getTable(identifier);
+            return 
Options.fromMap(table.options()).get(CoreOptions.LAKE_STREAM_ENABLED)
+                    ? table
+                    : null;
+        } catch (Catalog.TableNotExistException e) {
+            return null;
+        }
+    }
+
+    /** Whether the schema persisted at the connector's {@code path} has 
lake-stream mode on. */
+    private static boolean lakeStreamEnabledAtPath(DynamicTableFactory.Context 
context) {
+        Options options = 
Options.fromMap(context.getCatalogTable().getOptions());
+        String path = options.get(CoreOptions.PATH);
+        if (path == null) {
+            return options.get(CoreOptions.LAKE_STREAM_ENABLED);
+        }
+        // only the location goes in: the factory merges whatever it is given 
over the persisted
+        // schema, so passing the connector options would restate the very 
flag being read
+        Options location = new Options();
+        location.set(CoreOptions.PATH, path);
+        location.set(CoreOptions.BRANCH, options.get(CoreOptions.BRANCH));
+        Table table =
+                FileStoreTableFactory.create(
+                        CatalogContext.create(location, new 
FlinkFileIOLoader()));

Review Comment:
   [P2] Retain FileIO configuration while reading the persisted flag
   
   This guard now runs for every connector-mode sink with a path, including 
ordinary tables, but its CatalogContext keeps only path/branch and drops the 
connection/FileIO options from the connector mapping. I reproduced this with 
the existing SPI-registered IsolatedDirectoryFileIO: a plain persisted table 
and mapping with the required `root-dir` successfully create a sink through the 
complete-options factory route, while the normal HEAD route throws `Isolated 
file io requires option root-dir` here before sink creation. The same loss 
affects storage plugins requiring endpoint/credential options supplied through 
the mapping. Preserve those options for FileIO construction while reading 
lakestream.enabled directly from the persisted schema, so a connector override 
still cannot restate the persisted flag.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to