This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 79dae8ebb [doc] first-row merge engine adds changelog producer usage
documentation (#4298)
79dae8ebb is described below
commit 79dae8ebb479d5b3adab8f2ca39b4c838658f062
Author: Kerwin <[email protected]>
AuthorDate: Wed Oct 9 21:43:45 2024 +0800
[doc] first-row merge engine adds changelog producer usage documentation
(#4298)
---
docs/content/primary-key-table/merge-engine/aggregation.md | 2 +-
docs/content/primary-key-table/merge-engine/first-row.md | 9 +++++++--
docs/content/primary-key-table/merge-engine/overview.md | 2 +-
docs/content/primary-key-table/merge-engine/partial-update.md | 2 +-
.../src/main/java/org/apache/paimon/schema/SchemaValidation.java | 4 ++--
.../java/org/apache/paimon/table/PrimaryKeyFileStoreTable.java | 4 +---
.../src/test/java/org/apache/paimon/schema/TableSchemaTest.java | 2 +-
7 files changed, 14 insertions(+), 11 deletions(-)
diff --git a/docs/content/primary-key-table/merge-engine/aggregation.md
b/docs/content/primary-key-table/merge-engine/aggregation.md
index a9b9b5a38..fa667ed35 100644
--- a/docs/content/primary-key-table/merge-engine/aggregation.md
+++ b/docs/content/primary-key-table/merge-engine/aggregation.md
@@ -3,7 +3,7 @@ title: "Aggregation"
weight: 3
type: docs
aliases:
-- /cdc-ingestion/merge-engin/aggregation.html
+- /primary-key-table/merge-engin/aggregation.html
---
<!--
Licensed to the Apache Software Foundation (ASF) under one
diff --git a/docs/content/primary-key-table/merge-engine/first-row.md
b/docs/content/primary-key-table/merge-engine/first-row.md
index c726866c9..7e5501cf7 100644
--- a/docs/content/primary-key-table/merge-engine/first-row.md
+++ b/docs/content/primary-key-table/merge-engine/first-row.md
@@ -3,7 +3,7 @@ title: "First Row"
weight: 4
type: docs
aliases:
-- /cdc-ingestion/merge-engin/first-row.html
+- /primary-key-table/merge-engin/first-row.html
---
<!--
Licensed to the Apache Software Foundation (ASF) under one
@@ -29,11 +29,16 @@ under the License.
By specifying `'merge-engine' = 'first-row'`, users can keep the first row of
the same primary key. It differs from the
`deduplicate` merge engine that in the `first-row` merge engine, it will
generate insert only changelog.
+{{< hint info >}}
+`first-row` merge engine only supports `none` and `lookup` changelog producer.
+For streaming queries must be used with the `lookup` [changelog producer]({{<
ref "primary-key-table/changelog-producer" >}}).
+{{< /hint >}}
+
{{< hint info >}}
1. You can not specify [sequence.field]({{< ref
"primary-key-table/sequence-rowkind#sequence-field" >}}).
2. Not accept `DELETE` and `UPDATE_BEFORE` message. You can config
`ignore-delete` to ignore these two kinds records.
3. Visibility guarantee: Tables with First Row engine, the files with level 0
will only be visible after compaction.
So by default, compaction is synchronous, and if asynchronous is turned on,
there may be delays in the data.
- {{< /hint >}}
+{{< /hint >}}
This is of great help in replacing log deduplication in streaming computation.
diff --git a/docs/content/primary-key-table/merge-engine/overview.md
b/docs/content/primary-key-table/merge-engine/overview.md
index 17585a2df..9f3b50c39 100644
--- a/docs/content/primary-key-table/merge-engine/overview.md
+++ b/docs/content/primary-key-table/merge-engine/overview.md
@@ -3,7 +3,7 @@ title: "Overview"
weight: 1
type: docs
aliases:
-- /cdc-ingestion/merge-engin/overview.html
+- /primary-key-table/merge-engin/overview.html
---
<!--
Licensed to the Apache Software Foundation (ASF) under one
diff --git a/docs/content/primary-key-table/merge-engine/partial-update.md
b/docs/content/primary-key-table/merge-engine/partial-update.md
index 09c376c25..83b664c1b 100644
--- a/docs/content/primary-key-table/merge-engine/partial-update.md
+++ b/docs/content/primary-key-table/merge-engine/partial-update.md
@@ -3,7 +3,7 @@ title: "Partial Update"
weight: 2
type: docs
aliases:
- - /cdc-ingestion/merge-engin/partial-update.html
+- /primary-key-table/merge-engin/partial-update.html
---
<!--
diff --git
a/paimon-core/src/main/java/org/apache/paimon/schema/SchemaValidation.java
b/paimon-core/src/main/java/org/apache/paimon/schema/SchemaValidation.java
index c714b90d5..20a8787a4 100644
--- a/paimon-core/src/main/java/org/apache/paimon/schema/SchemaValidation.java
+++ b/paimon-core/src/main/java/org/apache/paimon/schema/SchemaValidation.java
@@ -180,7 +180,7 @@ public class SchemaValidation {
if (options.changelogProducer() != ChangelogProducer.LOOKUP
&& options.changelogProducer() != ChangelogProducer.NONE) {
throw new IllegalArgumentException(
- "Only support 'none' and 'lookup' changelog-producer
on FIRST_MERGE merge engine");
+ "Only support 'none' and 'lookup' changelog-producer
on FIRST_ROW merge engine");
}
}
@@ -540,7 +540,7 @@ public class SchemaValidation {
if (options.mergeEngine() == MergeEngine.FIRST_ROW) {
throw new IllegalArgumentException(
- "Do not support use sequence field on FIRST_MERGE
merge engine.");
+ "Do not support use sequence field on FIRST_ROW merge
engine.");
}
if (schema.crossPartitionUpdate()) {
diff --git
a/paimon-core/src/main/java/org/apache/paimon/table/PrimaryKeyFileStoreTable.java
b/paimon-core/src/main/java/org/apache/paimon/table/PrimaryKeyFileStoreTable.java
index 9c15a7bd1..b10cdbd06 100644
---
a/paimon-core/src/main/java/org/apache/paimon/table/PrimaryKeyFileStoreTable.java
+++
b/paimon-core/src/main/java/org/apache/paimon/table/PrimaryKeyFileStoreTable.java
@@ -29,7 +29,6 @@ import
org.apache.paimon.mergetree.compact.LookupMergeFunction;
import org.apache.paimon.mergetree.compact.MergeFunctionFactory;
import org.apache.paimon.operation.FileStoreScan;
import org.apache.paimon.operation.KeyValueFileStoreScan;
-import org.apache.paimon.options.Options;
import org.apache.paimon.predicate.Predicate;
import org.apache.paimon.schema.KeyValueFieldsExtractor;
import org.apache.paimon.schema.TableSchema;
@@ -72,8 +71,7 @@ class PrimaryKeyFileStoreTable extends AbstractFileStoreTable
{
public KeyValueFileStore store() {
if (lazyStore == null) {
RowType rowType = tableSchema.logicalRowType();
- Options conf = Options.fromMap(tableSchema.options());
- CoreOptions options = new CoreOptions(conf);
+ CoreOptions options = CoreOptions.fromMap(tableSchema.options());
KeyValueFieldsExtractor extractor =
PrimaryKeyTableUtils.PrimaryKeyFieldsExtractor.EXTRACTOR;
diff --git
a/paimon-core/src/test/java/org/apache/paimon/schema/TableSchemaTest.java
b/paimon-core/src/test/java/org/apache/paimon/schema/TableSchemaTest.java
index cf09d1ce4..d7f2e660a 100644
--- a/paimon-core/src/test/java/org/apache/paimon/schema/TableSchemaTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/schema/TableSchemaTest.java
@@ -154,7 +154,7 @@ public class TableSchemaTest {
options.put(MERGE_ENGINE.key(),
CoreOptions.MergeEngine.FIRST_ROW.toString());
assertThatThrownBy(() -> validateTableSchema(schema))
.hasMessageContaining(
- "Do not support use sequence field on FIRST_MERGE
merge engine.");
+ "Do not support use sequence field on FIRST_ROW merge
engine.");
options.put(FIELDS_PREFIX + ".f3." + AGG_FUNCTION, "max");
assertThatThrownBy(() -> validateTableSchema(schema))
