yunfengzhou-hub commented on code in PR #10833:
URL: https://github.com/apache/paimon/pull/10833#discussion_r4467567548


##########
paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lakestream/LakeStreamTableFactory.java:
##########
@@ -0,0 +1,336 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.flink.lakestream;
+
+import org.apache.paimon.CoreOptions;
+import org.apache.paimon.catalog.Catalog;
+import org.apache.paimon.catalog.CatalogContext;
+import org.apache.paimon.catalog.Identifier;
+import org.apache.paimon.flink.AbstractFlinkTableFactory;
+import org.apache.paimon.flink.FlinkCatalog;
+import org.apache.paimon.flink.FlinkCatalogFactory;
+import org.apache.paimon.flink.FlinkFileIOLoader;
+import org.apache.paimon.options.CatalogOptions;
+import org.apache.paimon.options.Options;
+import org.apache.paimon.table.FileStoreTableFactory;
+import org.apache.paimon.table.FormatTable;
+import org.apache.paimon.table.Table;
+
+import org.apache.flink.configuration.ConfigOption;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.configuration.ReadableConfig;
+import org.apache.flink.table.api.ValidationException;
+import org.apache.flink.table.catalog.CatalogBaseTable;
+import org.apache.flink.table.catalog.CatalogTable;
+import org.apache.flink.table.catalog.ObjectIdentifier;
+import org.apache.flink.table.catalog.ResolvedCatalogTable;
+import org.apache.flink.table.connector.sink.DynamicTableSink;
+import org.apache.flink.table.connector.source.DynamicTableSource;
+import org.apache.flink.table.factories.DynamicTableFactory;
+import org.apache.flink.table.factories.DynamicTableSinkFactory;
+import org.apache.flink.table.factories.DynamicTableSourceFactory;
+import org.apache.flink.table.factories.Factory;
+
+import javax.annotation.Nullable;
+
+import java.util.Collections;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+
+/**
+ * Builds the source and the sink of a lake-stream table by delegating to the 
Lakestream Store's own
+ * table factory. Not an SPI-registered factory: {@link #factoryIdentifier} is 
implemented only
+ * because the interface requires it.
+ */
+public class LakeStreamTableFactory implements DynamicTableSourceFactory, 
DynamicTableSinkFactory {
+
+    private final FlinkCatalog catalog;
+
+    public LakeStreamTableFactory(FlinkCatalog catalog) {
+        this.catalog = catalog;
+    }
+
+    @Override
+    public String factoryIdentifier() {
+        return FlinkCatalogFactory.IDENTIFIER;
+    }
+
+    @Override
+    public Set<ConfigOption<?>> requiredOptions() {
+        return Collections.emptySet();
+    }
+
+    @Override
+    public Set<ConfigOption<?>> optionalOptions() {
+        return Collections.emptySet();
+    }
+
+    /** Whether the read asks for the Lakestream Store's view of a table. */
+    public static boolean isLakeStreamRead(
+            @Nullable FlinkCatalog catalog, DynamicTableFactory.Context 
context) {
+        if (catalog == null) {
+            // with no catalog there is no lake-stream table to confirm the 
name against, so the
+            // name alone counts; the read is then refused for lack of a store 
rather than served
+            // from Paimon
+            return 
LakeStreamUtils.isRealTimeTable(context.getObjectIdentifier().toObjectPath());
+        }
+        return realTimeReadBaseTable(catalog.catalog(), context) != null;
+    }
+
+    /**
+     * Whether the write is to a lake-stream table, or through the read-only 
{@code $rt} name.
+     * Connector options and query hints do not change the answer.
+     */
+    public static boolean isLakeStreamWrite(
+            @Nullable FlinkCatalog catalog, DynamicTableFactory.Context 
context) {
+        Identifier identifier = identifierOf(context);
+        if (LakeStreamUtils.isRealTimeTable(identifier)) {
+            return true;
+        }
+        return catalog != null
+                ? persistedLakeStreamTable(catalog.catalog(), identifier) != 
null
+                : lakeStreamEnabledAtPath(context);
+    }
+
+    @Override
+    public DynamicTableSource 
createDynamicTableSource(DynamicTableFactory.Context context) {
+        Table baseTable = realTimeReadBaseTable(catalog.catalog(), context);
+        ObjectIdentifier baseIdentifier =
+                
LakeStreamUtils.stripRealTimeSuffix(context.getObjectIdentifier());
+        checkNotFormatTable(baseTable, baseIdentifier);
+        org.apache.flink.table.catalog.Catalog storeCatalog =
+                catalog.getOrCreateLakeStreamStoreCatalog();
+        // the catalog already resolved this read against the store, so the 
planner's table is the
+        // store's table with any query hints applied on top of it
+        CatalogBaseTable storeTable = context.getCatalogTable();
+        DynamicTableSourceFactory sourceFactory =
+                storeFactory(
+                        storeCatalog,
+                        DynamicTableSourceFactory.class,
+                        "create a source for '$rt'-suffixed reads");
+        return sourceFactory.createDynamicTableSource(
+                delegatedContext(context, baseIdentifier, storeTable));
+    }
+
+    @Override
+    public DynamicTableSink createDynamicTableSink(DynamicTableFactory.Context 
context) {
+        checkNotRealTimeWrite(context);
+        Table table = persistedLakeStreamTable(catalog.catalog(), 
identifierOf(context));
+        checkNotFormatTable(table, context.getObjectIdentifier());
+        ObjectIdentifier identifier = context.getObjectIdentifier();
+        org.apache.flink.table.catalog.Catalog storeCatalog =
+                catalog.getOrCreateLakeStreamStoreCatalog();
+        // the store table is fetched from the store catalog because the 
store's sink factory may
+        // require provider-specific table options (PIP-39 L289)
+        CatalogBaseTable storeTable =
+                catalog.requireLakeStreamStoreTable(identifier.toObjectPath());
+        DynamicTableSinkFactory sinkFactory =
+                storeFactory(
+                        storeCatalog, DynamicTableSinkFactory.class, "create a 
sink for writes");
+        return new LakeStreamDynamicTableSink(
+                sinkFactory.createDynamicTableSink(
+                        delegatedContext(context, identifier, storeTable)),
+                identifier);
+    }
+
+    /** A format table has no Paimon history for the store to tier into, so it 
cannot be one. */
+    private void checkNotFormatTable(@Nullable Table table, ObjectIdentifier 
identifier) {
+        if (table instanceof FormatTable) {
+            throw new ValidationException(
+                    String.format(
+                            "Table '%s' cannot be a lake-stream table: '%s' is 
only supported on "
+                                    + "Paimon tables, not on format tables.",
+                            identifier.asSummaryString(), 
CoreOptions.LAKE_STREAM_ENABLED.key()));
+        }
+    }
+
+    /** The base table of a {@code $rt} read, or {@code null} when the read 
stays on Paimon. */
+    @Nullable
+    private static Table realTimeReadBaseTable(
+            Catalog paimonCatalog, DynamicTableFactory.Context context) {
+        Identifier identifier = identifierOf(context);
+        if (!LakeStreamUtils.isRealTimeTable(identifier)) {
+            return null;
+        }
+        return persistedLakeStreamTable(
+                paimonCatalog,
+                Identifier.create(identifier.getDatabaseName(), 
identifier.getTableName()));
+    }
+
+    /** The Paimon table when lake-stream mode is enabled on it, otherwise 
{@code null}. */
+    @Nullable
+    private static Table persistedLakeStreamTable(Catalog paimonCatalog, 
Identifier identifier) {
+        try {
+            Table table = paimonCatalog.getTable(identifier);
+            return 
Options.fromMap(table.options()).get(CoreOptions.LAKE_STREAM_ENABLED)
+                    ? table
+                    : null;
+        } catch (Catalog.TableNotExistException e) {
+            return null;
+        }
+    }
+
+    /** Whether the schema persisted at the connector's {@code path} has 
lake-stream mode on. */
+    private static boolean lakeStreamEnabledAtPath(DynamicTableFactory.Context 
context) {
+        Options options = 
Options.fromMap(context.getCatalogTable().getOptions());
+        String path = options.get(CoreOptions.PATH);
+        if (path == null) {
+            return options.get(CoreOptions.LAKE_STREAM_ENABLED);
+        }
+        // only the location goes in: the factory merges whatever it is given 
over the persisted
+        // schema, so passing the connector options would restate the very 
flag being read
+        Options location = new Options();
+        location.set(CoreOptions.PATH, path);
+        location.set(CoreOptions.BRANCH, options.get(CoreOptions.BRANCH));
+        Table table =
+                FileStoreTableFactory.create(
+                        CatalogContext.create(location, new 
FlinkFileIOLoader()));
+        return 
Options.fromMap(table.options()).get(CoreOptions.LAKE_STREAM_ENABLED);
+    }
+
+    private static Identifier identifierOf(DynamicTableFactory.Context 
context) {
+        ObjectIdentifier identifier = context.getObjectIdentifier();
+        return new Identifier(identifier.getDatabaseName(), 
identifier.getObjectName());
+    }
+
+    /**
+     * The failure reported for a lake-stream table when no store was 
configured. Serving it from
+     * Paimon instead would leave the store and Paimon holding different 
values for the same keys.
+     *
+     * @param operation what the statement was trying to do, e.g. {@code 
"write to"}
+     */
+    public static ValidationException missingStoreError(
+            ObjectIdentifier identifier, String operation) {
+        return new ValidationException(
+                String.format(
+                        "Cannot %s lake-stream table '%s': the catalog was 
created without "
+                                + "'%s', so there is no Lakestream Store to 
serve it, and serving it "
+                                + "from Paimon instead would leave the store 
and Paimon holding "
+                                + "different data for the same keys. Create 
the catalog with '%s' and "
+                                + "the store's connection options, or set '%s' 
to false first.",
+                        operation,
+                        identifier.asSummaryString(),
+                        CatalogOptions.LAKE_STREAM_STORE.key(),
+                        CatalogOptions.LAKE_STREAM_STORE.key(),
+                        CoreOptions.LAKE_STREAM_ENABLED.key()));
+    }
+
+    /** Rejects a write through the read-only {@code $rt} name, which the 
store serves. */
+    private static void checkNotRealTimeWrite(DynamicTableFactory.Context 
context) {
+        ObjectIdentifier identifier = context.getObjectIdentifier();
+        if (LakeStreamUtils.isRealTimeTable(identifierOf(context))) {
+            throw new ValidationException(
+                    String.format(
+                            "Cannot write to '%s': the '$%s' suffix names the 
Lakestream "
+                                    + "Store's read-only view of a lake-stream 
table. Write to "
+                                    + "'%s' instead.",
+                            identifier.asSummaryString(),
+                            LakeStreamUtils.REAL_TIME_TABLE_SUFFIX,
+                            
LakeStreamUtils.stripRealTimeSuffix(identifier).asSummaryString()));
+        }
+    }
+
+    /** The store's factory of the given type, failing with an error naming 
the store. */
+    private <T> T storeFactory(
+            org.apache.flink.table.catalog.Catalog storeCatalog,
+            Class<T> factoryClass,
+            String purpose) {
+        String store = catalog.lakeStreamStoreIdentifier();
+        Optional<Factory> factory = storeCatalog.getFactory();
+        if (!factory.isPresent() || !factoryClass.isInstance(factory.get())) {
+            throw new ValidationException(
+                    String.format(
+                            "The Lakestream Store '%s' (configured via '%s' = 
'%s') does not "
+                                    + "provide a %s through its catalog 
factory, so Paimon cannot "
+                                    + "delegate lake-stream operations to it 
and cannot %s.",
+                            store,
+                            CatalogOptions.LAKE_STREAM_STORE.key(),
+                            store,
+                            factoryClass.getSimpleName(),
+                            purpose));
+        }
+        return factoryClass.cast(factory.get());
+    }
+
+    /**
+     * The planner's context adapted for the store's factory: the base 
identifier, the store table's
+     * options, and the store's catalog options after prefix removal.
+     */
+    private DynamicTableFactory.Context delegatedContext(
+            DynamicTableFactory.Context original,
+            ObjectIdentifier identifier,
+            CatalogBaseTable storeTable) {
+        if (!(storeTable instanceof CatalogTable)) {
+            throw new ValidationException(
+                    String.format(
+                            "Expected a table from the Lakestream Store '%s' 
for '%s', but got %s.",
+                            catalog.lakeStreamStoreIdentifier(),
+                            identifier.asSummaryString(),
+                            storeTable.getClass().getName()));
+        }
+        ResolvedCatalogTable resolvedStoreTable =
+                new ResolvedCatalogTable(
+                        (CatalogTable) storeTable, 
original.getCatalogTable().getResolvedSchema());

Review Comment:
   Fixed. Writes are now rejected before store sink creation unless the 
normalized Paimon and store schemas match in ordered physical fields and 
nullability, primary keys, partition keys, and shared immutable options. 
Store-private distribution settings and unrelated metadata are excluded.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to