This is an automated email from the ASF dual-hosted git repository.
luoyuxia pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/fluss.git
The following commit(s) were added to refs/heads/main by this push:
new dfa33de7c [docs] Reorganize Streaming Lakehouse documentation to
higlight deployment and core features (#2699)
dfa33de7c is described below
commit dfa33de7c589e8d05c13f79ce62f73bfd32ea629
Author: MehulBatra <[email protected]>
AuthorDate: Mon Jul 6 08:58:37 2026 +0530
[docs] Reorganize Streaming Lakehouse documentation to higlight deployment
and core features (#2699)
---------
Co-authored-by: luoyuxia <[email protected]>
---
website/build_versioned_docs.sh | 8 +
website/docs/engine-flink/getting-started.md | 4 +-
.../deploying-streaming-lakehouse.md | 250 +++++++++++++++++++++
website/docs/install-deploy/overview.md | 12 +-
website/docs/maintenance/operations/upgrading.md | 4 +-
.../maintenance/tiered-storage/_category_.json | 2 +-
.../filesystems/_category_.json | 2 +-
.../{ => tiered-storage}/filesystems/azure.md | 0
.../{ => tiered-storage}/filesystems/cos.md | 0
.../{ => tiered-storage}/filesystems/gs.md | 0
.../{ => tiered-storage}/filesystems/hdfs.md | 0
.../{ => tiered-storage}/filesystems/obs.md | 0
.../{ => tiered-storage}/filesystems/oss.md | 0
.../{ => tiered-storage}/filesystems/overview.md | 0
.../{ => tiered-storage}/filesystems/s3.md | 0
.../tiered-storage/lakehouse-storage.md | 137 ++++-------
.../maintenance/tiered-storage/remote-storage.md | 2 +-
website/docs/quickstart/flink.md | 4 +-
website/docs/quickstart/lakehouse.md | 4 +-
.../datalake-catalogs/_category_.json | 4 +
.../catalogs => datalake-catalogs}/lakekeeper.md | 14 +-
.../catalogs => datalake-catalogs}/polaris.md | 12 +-
.../datalake-formats/_category_.json | 4 +
.../formats => datalake-formats}/iceberg.md | 64 ++++--
.../formats => datalake-formats}/lance.md | 17 +-
.../formats => datalake-formats}/paimon.md | 12 +-
.../integrate-data-lakes/_category_.json | 4 -
.../integrate-data-lakes/catalogs/_category_.json | 4 -
.../integrate-data-lakes/formats/_category_.json | 4 -
website/docs/streaming-lakehouse/overview.md | 13 +-
.../docs/streaming-lakehouse/tiering-service.md | 102 +++++++++
website/docs/streaming-lakehouse/union-read.md | 113 ++++++++++
website/docs/table-design/table-types/log-table.md | 2 +-
website/docusaurus.config.ts | 31 ++-
34 files changed, 650 insertions(+), 179 deletions(-)
diff --git a/website/build_versioned_docs.sh b/website/build_versioned_docs.sh
index cca2f576a..f8a18293d 100755
--- a/website/build_versioned_docs.sh
+++ b/website/build_versioned_docs.sh
@@ -122,6 +122,14 @@ for branch in $branches; do
# Copy the website/docs directory to the target directory
$CP_CMD "$version_dir/" || { echo "Failed to copy for branch:
$clean_branch_name"; continue; }
echo "Copied documentation for branch '$clean_branch_name' to
'$version_dir'"
+
+ # Fix absolute links in versioned docs (convert
/docs/maintenance/filesystems/hdfs.md to relative path)
+
lakehouse_file="$version_dir/maintenance/tiered-storage/lakehouse-storage.md"
+ if [ -f "$lakehouse_file" ]; then
+ sed -i.bak
's|/docs/maintenance/filesystems/hdfs\.md|../filesystems/hdfs.md|g'
"$lakehouse_file"
+ rm -f "$lakehouse_file.bak"
+ echo "Fixed absolute links in '$lakehouse_file'"
+ fi
else
echo "The website/docs directory does not exist in branch
'$clean_branch_name', skipping..."
fi
diff --git a/website/docs/engine-flink/getting-started.md
b/website/docs/engine-flink/getting-started.md
index 44d3fe9fe..130851a1c 100644
--- a/website/docs/engine-flink/getting-started.md
+++ b/website/docs/engine-flink/getting-started.md
@@ -85,8 +85,8 @@ Download the Fluss Flink connector JAR from the
[Dependencies](#dependencies) se
cp fluss-flink-1.20-$FLUSS_VERSION$.jar <FLINK_HOME>/lib/
```
:::note
-If you use [Amazon S3](http://aws.amazon.com/s3/), [Aliyun
OSS](https://www.aliyun.com/product/oss) or [HDFS(Hadoop Distributed File
System)](https://hadoop.apache.org/docs/stable/) as Fluss's [remote
storage](maintenance/tiered-storage/remote-storage.md),
-you should download the corresponding Fluss filesystem JAR (see
[Filesystems](../maintenance/filesystems/overview.md)) and also copy it to the
lib directory of your Flink home.
+If you use [Amazon S3](http://aws.amazon.com/s3/), [Aliyun
OSS](https://www.aliyun.com/product/oss) or [HDFS(Hadoop Distributed File
System)](https://hadoop.apache.org/docs/stable/) as Fluss's [remote
storage](../maintenance/tiered-storage/remote-storage.md),
+you should download the corresponding [Fluss filesystem
jar](/downloads#filesystem-jars) and also copy it to the lib directory of your
Flink home.
:::
- **Start a local cluster**
diff --git a/website/docs/install-deploy/deploying-streaming-lakehouse.md
b/website/docs/install-deploy/deploying-streaming-lakehouse.md
new file mode 100644
index 000000000..216248ebe
--- /dev/null
+++ b/website/docs/install-deploy/deploying-streaming-lakehouse.md
@@ -0,0 +1,250 @@
+---
+title: "Deploying Streaming Lakehouse"
+sidebar_position: 6
+---
+
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# Deploying Streaming Lakehouse
+
+This guide covers how to deploy a Fluss cluster with Streaming Lakehouse
capabilities. For conceptual overview, see [Lakehouse
Overview](../streaming-lakehouse/overview.md).
+
+## Prerequisites
+
+1. A running Fluss cluster (see [Deploying Distributed
Cluster](deploying-distributed-cluster.md))
+2. A running Flink cluster (for the Tiering Service)
+3. Access to a data lake storage system (S3, HDFS, OSS, etc.)
+
+## Cluster Configuration
+
+You can enable Lakehouse storage through:
+1. **Static configuration**: Configure in `server.yaml` before starting the
cluster
+2. **Dynamic configuration**: Enable at runtime using the
`set_cluster_configs` procedure
+
+### Method 1: Static Configuration
+
+Configure lakehouse settings in `server.yaml` on all Fluss servers
(CoordinatorServer and TabletServer).
+
+Fluss follows a simple convention for these keys:
+
+- `datalake.enabled: true` explicitly enables lakehouse capability for the
cluster. If it is left unset, configuring `datalake.format` alone also enables
it; whenever `datalake.enabled` is `true`, `datalake.format` must also be set.
+- `datalake.format` selects the lake format for the cluster (`paimon`,
`iceberg`, or `lance`).
+- Format-specific options use the `datalake.<format>.*` prefix. Fluss strips
this prefix and passes the remaining keys straight to the corresponding lake
catalog/client — for example `datalake.paimon.metastore` becomes `metastore`
and `datalake.iceberg.type` becomes `type`. Any option supported by the lake
catalog can be set this way.
+- Only configure options for the format selected by `datalake.format`.
+
+:::note
+The [Tiering Service](../streaming-lakehouse/tiering-service.md) is an
independent Flink job and needs the same `datalake.*` options passed as job
arguments (see [Starting the Tiering Service](#starting-the-tiering-service)).
+:::
+
+<Tabs groupId="datalake-format">
+<TabItem value="paimon" label="Paimon" default>
+
+```yaml title="server.yaml"
+datalake.enabled: true
+datalake.format: paimon
+datalake.paimon.metastore: filesystem
+datalake.paimon.warehouse: /path/to/paimon/warehouse
+```
+
+For Hive catalog:
+```yaml title="server.yaml"
+datalake.enabled: true
+datalake.format: paimon
+datalake.paimon.metastore: hive
+datalake.paimon.uri: thrift://<hive-metastore-host>:<port>
+datalake.paimon.warehouse: hdfs:///path/to/warehouse
+```
+
+</TabItem>
+<TabItem value="iceberg" label="Iceberg">
+
+```yaml title="server.yaml"
+datalake.enabled: true
+datalake.format: iceberg
+datalake.iceberg.catalog-impl: org.apache.iceberg.jdbc.JdbcCatalog
+datalake.iceberg.name: fluss_catalog
+datalake.iceberg.uri: jdbc:postgresql://postgres-host:5432/iceberg
+datalake.iceberg.jdbc.user: iceberg
+datalake.iceberg.jdbc.password: iceberg
+datalake.iceberg.warehouse: s3://bucket/iceberg
+datalake.iceberg.io-impl: org.apache.iceberg.aws.s3.S3FileIO
+```
+
+</TabItem>
+<TabItem value="lance" label="Lance">
+
+```yaml title="server.yaml"
+datalake.enabled: true
+datalake.format: lance
+datalake.lance.warehouse: s3://bucket/lance
+```
+
+</TabItem>
+</Tabs>
+
+### Method 2: Dynamic Configuration
+
+Enable lakehouse settings at runtime using Flink SQL:
+
+```sql title="Flink SQL"
+USE fluss_catalog;
+
+CALL sys.set_cluster_configs(
+ config_pairs => 'datalake.format', 'paimon',
+ 'datalake.paimon.metastore', 'filesystem',
+ 'datalake.paimon.warehouse', '/path/to/warehouse'
+);
+```
+
+See [set_cluster_configs](../engine-flink/procedures.md#set_cluster_configs)
for more details.
+
+## Adding Required JARs
+
+### Fluss Server JARs
+
+Add JARs to `${FLUSS_HOME}/plugins/<format>/` based on your configuration:
+
+<Tabs groupId="datalake-format">
+<TabItem value="paimon" label="Paimon" default>
+
+| Scenario | Required JAR |
+|----------|--------------|
+| Paimon with S3 |
[`paimon-s3-<version>.jar`](https://paimon.apache.org/docs/$PAIMON_VERSION_SHORT$/project/download/),
matching your Paimon version |
+| Paimon with OSS |
[`paimon-oss-<version>.jar`](https://paimon.apache.org/docs/$PAIMON_VERSION_SHORT$/project/download/),
matching your Paimon version |
+| Paimon Hive catalog | [Flink SQL Hive connector
JAR](https://nightlies.apache.org/flink/flink-docs-stable/docs/connectors/table/hive/overview/#using-bundled-hive-jar)
|
+
+</TabItem>
+<TabItem value="iceberg" label="Iceberg">
+
+| Scenario | Required JAR |
+|----------|--------------|
+| Iceberg with S3 | [`iceberg-aws-<version>.jar`,
`iceberg-aws-bundle-<version>.jar`](https://iceberg.apache.org/docs/1.10.1/aws/),
matching your Iceberg version |
+| Iceberg JDBC catalog | PostgreSQL/MySQL JDBC driver |
+
+</TabItem>
+<TabItem value="lance" label="Lance">
+
+Lance support is built into the Fluss distribution. Cloud storage credentials
are configured via storage-options.
+
+</TabItem>
+</Tabs>
+
+## Starting the Tiering Service
+
+The Tiering Service is a Flink job that continuously tiers data from Fluss to
the data lake. For architecture details, see [Tiering
Service](../streaming-lakehouse/tiering-service.md).
+
+### Prerequisites
+
+1. Download
[fluss-flink-tiering-$FLUSS_VERSION$.jar](https://repo1.maven.org/maven2/org/apache/fluss/fluss-flink-tiering/$FLUSS_VERSION$/fluss-flink-tiering-$FLUSS_VERSION$.jar)
+
+### Flink JARs
+
+Add the following to `${FLINK_HOME}/lib`:
+
+<Tabs groupId="datalake-format">
+<TabItem value="paimon" label="Paimon" default>
+
+-
[fluss-flink-1.20-$FLUSS_VERSION$.jar](https://repo1.maven.org/maven2/org/apache/fluss/fluss-flink-1.20/$FLUSS_VERSION$/fluss-flink-1.20-$FLUSS_VERSION$.jar)
+-
[fluss-lake-paimon-$FLUSS_VERSION$.jar](https://repo1.maven.org/maven2/org/apache/fluss/fluss-lake-paimon/$FLUSS_VERSION$/fluss-lake-paimon-$FLUSS_VERSION$.jar)
+-
[paimon-bundle-$PAIMON_VERSION$.jar](https://repo.maven.apache.org/maven2/org/apache/paimon/paimon-bundle/$PAIMON_VERSION$/paimon-bundle-$PAIMON_VERSION$.jar)
+- [flink-shaded-hadoop-2-uber-*.jar](https://flink.apache.org/downloads/)
+- Paimon filesystem JAR (e.g., `paimon-s3-<version>.jar` for S3)
+
+</TabItem>
+<TabItem value="iceberg" label="Iceberg">
+
+-
[fluss-flink-1.20-$FLUSS_VERSION$.jar](https://repo1.maven.org/maven2/org/apache/fluss/fluss-flink-1.20/$FLUSS_VERSION$/fluss-flink-1.20-$FLUSS_VERSION$.jar)
+-
[fluss-lake-iceberg-$FLUSS_VERSION$.jar](https://repo1.maven.org/maven2/org/apache/fluss/fluss-lake-iceberg/$FLUSS_VERSION$/fluss-lake-iceberg-$FLUSS_VERSION$.jar)
+-
[iceberg-flink-runtime-1.20-*.jar](https://repo1.maven.org/maven2/org/apache/iceberg/iceberg-flink-runtime-1.20/)
+- Hadoop client JARs — export `HADOOP_CLASSPATH`, download the pre-bundled
[`hadoop-apache-3.3.5-2.jar`](https://repo1.maven.org/maven2/io/trino/hadoop/hadoop-apache/3.3.5-2/hadoop-apache-3.3.5-2.jar),
or install a full Hadoop package (see [Iceberg Hadoop
Dependencies](../streaming-lakehouse/datalake-formats/iceberg.md#prerequisites-hadoop-dependencies))
+- JDBC driver (if using JDBC catalog)
+
+</TabItem>
+<TabItem value="lance" label="Lance">
+
+-
[fluss-flink-1.20-$FLUSS_VERSION$.jar](https://repo1.maven.org/maven2/org/apache/fluss/fluss-flink-1.20/$FLUSS_VERSION$/fluss-flink-1.20-$FLUSS_VERSION$.jar)
+-
[fluss-lake-lance-$FLUSS_VERSION$.jar](https://repo1.maven.org/maven2/org/apache/fluss/fluss-lake-lance/$FLUSS_VERSION$/fluss-lake-lance-$FLUSS_VERSION$.jar)
+
+</TabItem>
+</Tabs>
+
+If using S3, OSS, or HDFS as Fluss's [remote
storage](../maintenance/tiered-storage/remote-storage.md), also add the
corresponding [Fluss filesystem JAR](/downloads#filesystem-jars).
+
+### Start the Service
+
+<Tabs groupId="datalake-format">
+<TabItem value="paimon" label="Paimon" default>
+
+```shell
+${FLINK_HOME}/bin/flink run /path/to/fluss-flink-tiering-$FLUSS_VERSION$.jar \
+ --fluss.bootstrap.servers localhost:9123 \
+ --datalake.format paimon \
+ --datalake.paimon.metastore filesystem \
+ --datalake.paimon.warehouse /tmp/paimon
+```
+
+</TabItem>
+<TabItem value="iceberg" label="Iceberg">
+
+```shell
+${FLINK_HOME}/bin/flink run /path/to/fluss-flink-tiering-$FLUSS_VERSION$.jar \
+ --fluss.bootstrap.servers localhost:9123 \
+ --datalake.format iceberg \
+ --datalake.iceberg.catalog-impl org.apache.iceberg.jdbc.JdbcCatalog \
+ --datalake.iceberg.name fluss_catalog \
+ --datalake.iceberg.uri "jdbc:postgresql://postgres:5432/iceberg" \
+ --datalake.iceberg.jdbc.user iceberg \
+ --datalake.iceberg.jdbc.password iceberg \
+ --datalake.iceberg.warehouse "s3://bucket/iceberg"
+```
+
+</TabItem>
+<TabItem value="lance" label="Lance">
+
+```shell
+${FLINK_HOME}/bin/flink run /path/to/fluss-flink-tiering-$FLUSS_VERSION$.jar \
+ --fluss.bootstrap.servers localhost:9123 \
+ --datalake.format lance \
+ --datalake.lance.warehouse s3://bucket/lance
+```
+
+</TabItem>
+</Tabs>
+
+:::note
+- You must pass all `datalake.*` options that were set in `server.yaml` as
command-line arguments
+- For S3/cloud storage, include additional flags like
`--datalake.paimon.s3.endpoint`, `--datalake.paimon.s3.access-key`, etc.
+- The Tiering Service is stateless—you can run multiple instances for
scalability
+- Use `-D` to pass Flink configurations (e.g., `-Dparallelism.default=3`)
+- For complete examples with S3 configuration, see the [Lakehouse
Quickstart](../quickstart/lakehouse.md)
+:::
+
+## Enabling Lakehouse for Tables
+
+Create tables with lakehouse storage enabled:
+
+```sql title="Flink SQL"
+CREATE TABLE my_table (
+ id BIGINT PRIMARY KEY NOT ENFORCED,
+ name STRING
+) WITH (
+ 'table.datalake.enabled' = 'true',
+ 'table.datalake.freshness' = '1min'
+);
+```
+
+## Verification
+
+1. Check the Tiering Service job is running in Flink Web UI
+2. After the freshness interval, query the lake table:
+
+```sql title="Flink SQL"
+-- Lake-only query
+SELECT * FROM my_table$lake;
+
+-- Union Read (real-time + historical)
+SELECT * FROM my_table;
+```
+
+See [Union Read](../streaming-lakehouse/union-read.md) for more details.
diff --git a/website/docs/install-deploy/overview.md
b/website/docs/install-deploy/overview.md
index a6c46b782..48784adb1 100644
--- a/website/docs/install-deploy/overview.md
+++ b/website/docs/install-deploy/overview.md
@@ -103,9 +103,9 @@ We have listed them in the table below the figure.
Fluss uses file systems as remote storage to store snapshots
for Primary-Key Table and store tiered log segments for Log Table.
</td>
<td>
- <li>[HDFS](maintenance/filesystems/hdfs.md)</li>
- <li>[Aliyun OSS](maintenance/filesystems/oss.md)</li>
- <li>[Amazon S3](maintenance/filesystems/s3.md)</li>
+ <li>[HDFS](../maintenance/tiered-storage/filesystems/hdfs.md)</li>
+ <li>[Aliyun
OSS](../maintenance/tiered-storage/filesystems/oss.md)</li>
+ <li>[Amazon
S3](../maintenance/tiered-storage/filesystems/s3.md)</li>
</td>
</tr>
<tr>
@@ -116,9 +116,9 @@ We have listed them in the table below the figure.
by query engines such as Flink, Spark, StarRocks, Trino.
</td>
<td>
-
<li>[Paimon](streaming-lakehouse/integrate-data-lakes/formats/paimon.md)</li>
-
<li>[Iceberg](streaming-lakehouse/integrate-data-lakes/formats/iceberg.md)</li>
-
<li>[Lance](streaming-lakehouse/integrate-data-lakes/formats/lance.md)</li>
+
<li>[Paimon](../streaming-lakehouse/datalake-formats/paimon.md)</li>
+
<li>[Iceberg](../streaming-lakehouse/datalake-formats/iceberg.md)</li>
+
<li>[Lance](../streaming-lakehouse/datalake-formats/lance.md)</li>
</td>
</tr>
<tr>
diff --git a/website/docs/maintenance/operations/upgrading.md
b/website/docs/maintenance/operations/upgrading.md
index 1eb768a46..b1312f033 100644
--- a/website/docs/maintenance/operations/upgrading.md
+++ b/website/docs/maintenance/operations/upgrading.md
@@ -34,8 +34,8 @@ tar -xzf fluss-$FLUSS_VERSION$-bin.tgz
cd fluss-$FLUSS_VERSION$/
```
-2. If you want to enable [Lakehouse
Storage](docs/maintenance/tiered-storage/lakehouse-storage.md), you need to
prepare the required JAR files for the datalake first. For more details,
- see [Add other jars required by
datalake](docs/maintenance/tiered-storage/lakehouse-storage.md#add-other-jars-required-by-datalake).
+2. If you want to enable [Lakehouse
Storage](../tiered-storage/lakehouse-storage.md), you need to prepare the
required JAR files for the datalake first. For more details,
+ see [Deploying Streaming
Lakehouse](../../install-deploy/deploying-streaming-lakehouse.md).
3. Next, copy the configuration options from 0.6
(`fluss-0.6/conf/server.yaml`) to the new configuration
file (`fluss-$FLUSS_VERSION$/conf/server.yaml`). Adding any new options
introduced in version $FLUSS_VERSION$ as
diff --git a/website/docs/maintenance/tiered-storage/_category_.json
b/website/docs/maintenance/tiered-storage/_category_.json
index 0c5af719d..dc54765ee 100644
--- a/website/docs/maintenance/tiered-storage/_category_.json
+++ b/website/docs/maintenance/tiered-storage/_category_.json
@@ -1,4 +1,4 @@
{
"label": "Tiered Storage",
- "position": 3
+ "position": 2
}
diff --git a/website/docs/maintenance/filesystems/_category_.json
b/website/docs/maintenance/tiered-storage/filesystems/_category_.json
similarity index 65%
rename from website/docs/maintenance/filesystems/_category_.json
rename to website/docs/maintenance/tiered-storage/filesystems/_category_.json
index 02fd650d3..55a5c75be 100644
--- a/website/docs/maintenance/filesystems/_category_.json
+++ b/website/docs/maintenance/tiered-storage/filesystems/_category_.json
@@ -1,4 +1,4 @@
{
"label": "File Systems",
- "position": 2
+ "position": 4
}
diff --git a/website/docs/maintenance/filesystems/azure.md
b/website/docs/maintenance/tiered-storage/filesystems/azure.md
similarity index 100%
rename from website/docs/maintenance/filesystems/azure.md
rename to website/docs/maintenance/tiered-storage/filesystems/azure.md
diff --git a/website/docs/maintenance/filesystems/cos.md
b/website/docs/maintenance/tiered-storage/filesystems/cos.md
similarity index 100%
rename from website/docs/maintenance/filesystems/cos.md
rename to website/docs/maintenance/tiered-storage/filesystems/cos.md
diff --git a/website/docs/maintenance/filesystems/gs.md
b/website/docs/maintenance/tiered-storage/filesystems/gs.md
similarity index 100%
rename from website/docs/maintenance/filesystems/gs.md
rename to website/docs/maintenance/tiered-storage/filesystems/gs.md
diff --git a/website/docs/maintenance/filesystems/hdfs.md
b/website/docs/maintenance/tiered-storage/filesystems/hdfs.md
similarity index 100%
rename from website/docs/maintenance/filesystems/hdfs.md
rename to website/docs/maintenance/tiered-storage/filesystems/hdfs.md
diff --git a/website/docs/maintenance/filesystems/obs.md
b/website/docs/maintenance/tiered-storage/filesystems/obs.md
similarity index 100%
rename from website/docs/maintenance/filesystems/obs.md
rename to website/docs/maintenance/tiered-storage/filesystems/obs.md
diff --git a/website/docs/maintenance/filesystems/oss.md
b/website/docs/maintenance/tiered-storage/filesystems/oss.md
similarity index 100%
rename from website/docs/maintenance/filesystems/oss.md
rename to website/docs/maintenance/tiered-storage/filesystems/oss.md
diff --git a/website/docs/maintenance/filesystems/overview.md
b/website/docs/maintenance/tiered-storage/filesystems/overview.md
similarity index 100%
rename from website/docs/maintenance/filesystems/overview.md
rename to website/docs/maintenance/tiered-storage/filesystems/overview.md
diff --git a/website/docs/maintenance/filesystems/s3.md
b/website/docs/maintenance/tiered-storage/filesystems/s3.md
similarity index 100%
rename from website/docs/maintenance/filesystems/s3.md
rename to website/docs/maintenance/tiered-storage/filesystems/s3.md
diff --git a/website/docs/maintenance/tiered-storage/lakehouse-storage.md
b/website/docs/maintenance/tiered-storage/lakehouse-storage.md
index 223037812..a9174073d 100644
--- a/website/docs/maintenance/tiered-storage/lakehouse-storage.md
+++ b/website/docs/maintenance/tiered-storage/lakehouse-storage.md
@@ -5,16 +5,11 @@ sidebar_position: 3
# Lakehouse Storage
-Lakehouse represents a new, open architecture that combines the best elements
of data lakes and data warehouses.
-Lakehouse combines data lake scalability and cost-effectiveness with data
warehouse reliability and performance.
+Fluss leverages well-known Lakehouse storage solutions like Apache Paimon,
Apache Iceberg, and Lance as the tiered storage layer. The Tiering Service
continuously tiers Fluss data to Lakehouse storage, where it can be read by
Fluss clients in a streaming manner and accessed directly by external systems
such as Flink, Spark, StarRocks, and others.
-Fluss leverages the well-known Lakehouse storage solutions like Apache Paimon,
Apache Iceberg, Apache Hudi, Delta Lake as
-the tiered storage layer. Currently, only Apache Paimon, Apache Iceberg, Lance
are supported, with more kinds of Lakehouse storage support are on the way.
-
-Fluss's datalake tiering service will tier Fluss's data to the Lakehouse
storage continuously. The data in Lakehouse storage can be read both by Fluss's
client in a streaming manner and accessed directly
-by external systems such as Flink, Spark, StarRocks and others. With data
tiered in Lakehouse storage, Fluss
-can gain much storage cost reduction and analytics performance improvement.
+For deployment instructions, see [Deploying Streaming
Lakehouse](../../install-deploy/deploying-streaming-lakehouse.md).
+For architecture details, see [Tiering
Service](../../streaming-lakehouse/tiering-service.md).
## Dependencies
@@ -34,104 +29,56 @@ Maven coordinates:
</dependency>
```
-Verify downloaded JARs against the [KEYS
file](https://downloads.apache.org/incubator/fluss/KEYS) using the
[verification instructions](/downloads).
-
-
-## Enable Lakehouse Storage
+Verify downloaded JARs against the [KEYS
file](https://downloads.apache.org/incubator/fluss/KEYS) using the
[verification instructions](/downloads#verifying-downloads).
-Lakehouse Storage is disabled by default, you must enable it manually.
+## Cluster Configuration
-The following example uses Paimon for demonstration—other data lake formats
follow similar steps, but require different configuration settings and JAR
files.
-You can refer to the documentation of the corresponding data lake format
integration for required configurations and JAR files.
+Lakehouse storage is configured in `server.yaml` using the `datalake.` prefix:
-### Lakehouse Storage Cluster Configurations
-#### Modify `server.yaml`
-First, you must configure the lakehouse storage in `server.yaml`. Take Paimon
as an example, you must configure the following configurations:
-```yaml
-# Paimon configuration
-datalake.enabled: true
+```yaml title="server.yaml"
datalake.format: paimon
-
-# the catalog config about Paimon, assuming using Filesystem catalog
datalake.paimon.metastore: filesystem
datalake.paimon.warehouse: /tmp/paimon
```
-Fluss processes Paimon configurations by removing the `datalake.paimon.`
prefix and then use the remaining configuration (without the prefix
`datalake.paimon.`) to create the Paimon catalog. Checkout the [Paimon
documentation](https://paimon.apache.org/docs/$PAIMON_VERSION_SHORT$/maintenance/configurations/)
for more details on the available configurations.
-
-For example, if you want to configure to use Hive catalog, you can configure
like following:
-```yaml
-datalake.enabled: true
-datalake.format: paimon
-datalake.paimon.metastore: hive
-datalake.paimon.uri: thrift://<hive-metastore-host-name>:<port>
-datalake.paimon.warehouse: hdfs:///path/to/warehouse
-```
-
-#### Add other jars required by datalake
-While Fluss includes the core Paimon library, additional jars may still need
to be manually added to `${FLUSS_HOME}/plugins/paimon/` according to your needs.
-For example:
-- If you are using Paimon filesystem catalog with OSS filesystem, you need to
put `paimon-oss-<paimon_version>.jar` into directory
`${FLUSS_HOME}/plugins/paimon/`.
-- If you are using Paimon Hive catalog, you need to put [the flink sql hive
connector
jar](https://nightlies.apache.org/flink/flink-docs-stable/docs/connectors/table/hive/overview/#using-bundled-hive-jar)
into directory `${FLUSS_HOME}/plugins/paimon/`.
-
-Additionally, when using Paimon with HDFS, you must also configure the Fluss
server with the Hadoop environment. See the [HDFS setup
guide](/docs/maintenance/filesystems/hdfs.md) for detailed instructions.
-
-### Start The Datalake Tiering Service
-Then, you must start the datalake tiering service to tier Fluss's data to the
lakehouse storage.
-#### Prerequisites
-- A running Flink cluster (currently only Flink is supported as the tiering
backend)
-- Download
[fluss-flink-tiering-$FLUSS_VERSION$.jar]($FLUSS_MAVEN_REPO_URL$/org/apache/fluss/fluss-flink-tiering/$FLUSS_VERSION$/fluss-flink-tiering-$FLUSS_VERSION$.jar)
(see [Dependencies](#dependencies))
-
-#### Prepare required jars
-- Put the Fluss Flink connector JAR into `${FLINK_HOME}/lib`; pick the version
matching your Flink runtime (see
[Dependencies](../../engine-flink/getting-started.md#dependencies)). For Flink
1.20, use
[fluss-flink-1.20-$FLUSS_VERSION$.jar]($FLUSS_MAVEN_REPO_URL$/org/apache/fluss/fluss-flink-1.20/$FLUSS_VERSION$/fluss-flink-1.20-$FLUSS_VERSION$.jar).
-- If you are using [Amazon S3](http://aws.amazon.com/s3/), [Aliyun
OSS](https://www.aliyun.com/product/oss) or [HDFS(Hadoop Distributed File
System)](https://hadoop.apache.org/docs/stable/) as Fluss's [remote
storage](maintenance/tiered-storage/remote-storage.md),
- you should download the corresponding Fluss filesystem JAR (see
[Filesystems](../filesystems/overview.md)) and also put it into
`${FLINK_HOME}/lib`
-- Put
[fluss-lake-paimon-$FLUSS_VERSION$.jar]($FLUSS_MAVEN_REPO_URL$/org/apache/fluss/fluss-lake-paimon/$FLUSS_VERSION$/fluss-lake-paimon-$FLUSS_VERSION$.jar)
into `${FLINK_HOME}/lib`
-- Put [paimon-bundle
jar](https://repo.maven.apache.org/maven2/org/apache/paimon/paimon-bundle/$PAIMON_VERSION$/paimon-bundle-$PAIMON_VERSION$.jar)
into `${FLINK_HOME}/lib`
-- [Download](https://flink.apache.org/downloads/) pre-bundled Hadoop jar
`flink-shaded-hadoop-2-uber-*.jar` and put into `${FLINK_HOME}/lib`
-- Put Paimon's [filesystem
jar](https://paimon.apache.org/docs/$PAIMON_VERSION_SHORT$/project/download/)
into `${FLINK_HOME}/lib`, if you use s3 to store paimon data, please put
`paimon-s3` jar into `${FLINK_HOME}/lib`
-- The other jars that Paimon may require, for example, if you use HiveCatalog,
you will need to put hive related jars
-
-
-#### Start Datalake Tiering Service
-After the Flink Cluster has been started, you can execute the
`fluss-flink-tiering-$FLUSS_VERSION$.jar` by using the following command to
start datalake tiering service:
-```shell
-<FLINK_HOME>/bin/flink run /path/to/fluss-flink-tiering-$FLUSS_VERSION$.jar \
- --fluss.bootstrap.servers localhost:9123 \
- --datalake.format paimon \
- --datalake.paimon.metastore filesystem \
- --datalake.paimon.warehouse /tmp/paimon
-```
-
-**Note:**
-- The `fluss.bootstrap.servers` should be the bootstrap server address of your
Fluss cluster. You must configure all options with the `datalake.` prefix in
the [server.yaml](#modify-serveryaml) file to run the tiering service. In this
case, these parameters are `--datalake.format`, `--datalake.paimon.metastore`,
and `--datalake.paimon.warehouse`.
-- The Flink tiering service is stateless, and you can run multiple tiering
services simultaneously to tier tables in Fluss.
-These tiering services are coordinated by the Fluss cluster to ensure
exactly-once semantics when tiering data to the lake storage. This means you
can freely scale the service up or down according to your workload.
-- This follows the standard practice for [submitting jobs to
Flink](https://nightlies.apache.org/flink/flink-docs-release-1.20/docs/deployment/cli/),
where you can use the `-D` parameter to specify Flink-related configurations.
-For example, if you want to set the tiering service job name to `My Fluss
Tiering Service1` and use `3` as the job parallelism, you can use the following
command:
-```shell
-<FLINK_HOME>/bin/flink run \
- -Dpipeline.name="My Fluss Tiering Service1" \
- -Dparallelism.default=3 \
- /path/to/fluss-flink-tiering-$FLUSS_VERSION$.jar \
- --fluss.bootstrap.servers localhost:9123 \
- --datalake.format paimon \
- --datalake.paimon.metastore filesystem \
- --datalake.paimon.warehouse /tmp/paimon
+Fluss processes configurations by removing the `datalake.<format>.` prefix and
uses the remaining configuration to create the data lake catalog.
+
+For format-specific configuration, see:
+- [Paimon](../../streaming-lakehouse/datalake-formats/paimon.md)
+- [Iceberg](../../streaming-lakehouse/datalake-formats/iceberg.md)
+- [Lance](../../streaming-lakehouse/datalake-formats/lance.md)
+
+## Table-Level Options
+
+| Option | Type | Default | Description |
+|--------|------|---------|-------------|
+| `table.datalake.enabled` | Boolean | false | Enable lakehouse storage for
this table |
+| `table.datalake.freshness` | Duration | 3min | Maximum lag between Fluss and
data lake table |
+| `table.datalake.format` | String | - | Data lake format (paimon, iceberg,
lance). Inherits from cluster config |
+| `table.datalake.auto-compaction` | Boolean | false | Auto-trigger compaction
in the data lake |
+| `table.datalake.auto-expire-snapshot` | Boolean | false | Auto-expire
snapshots in the data lake |
+
+Example:
+```sql title="Flink SQL"
+CREATE TABLE my_table (
+ id BIGINT PRIMARY KEY NOT ENFORCED,
+ name STRING
+) WITH (
+ 'table.datalake.enabled' = 'true',
+ 'table.datalake.freshness' = '1min'
+);
```
-### Enable Lakehouse Storage Per Table
-To enable lakehouse storage for a table, the table must be created with the
option `'table.datalake.enabled' = 'true'`.
+## Tiering Service Options
-Another option `table.datalake.freshness`, allows per-table configuration of
data freshness in the datalake.
-It defines the maximum amount of time that the datalake table's content should
lag behind updates to the Fluss table.
-Based on this target freshness, the Fluss tiering service automatically moves
data from the Fluss table and updates to the datalake table, so that the data
in the datalake table is kept up to date within this target.
-The default is `3min`, if the data does not need to be as fresh, you can
specify a longer target freshness time to reduce costs.
+| Option | Type | Default | Description |
+|--------|------|---------|-------------|
+| `lake.tiering.auto-expire-snapshot` | Boolean | false | Auto-trigger
snapshot expiration on commit, even if `table.datalake.auto-expire-snapshot` is
false |
-# Datalake Tiering Service Options
+## Data Retention
-The following table lists the options that can be used to configure the
datalake tiering service.
+When using lakehouse storage with Union Read:
+- **Expired Fluss log data** (controlled by `table.log.ttl`) remains
accessible via the lake if previously tiered
+- **Cleaned-up partitions** (controlled by
`table.auto-partition.num-retention`) remain accessible via the lake if
previously tiered
-| Option | Type | Default | Description
|
-|-----------------------------------------|----------|---------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| lake.tiering.auto-expire-snapshot | Boolean | false | If true,
snapshot expiration will be triggered automatically when tiering service
commits to the datalake, even if `table.datalake.auto-expire-snapshot` is
false. |
\ No newline at end of file
+This enables Fluss to store only recent data while the lake serves as a
complete historical archive.
diff --git a/website/docs/maintenance/tiered-storage/remote-storage.md
b/website/docs/maintenance/tiered-storage/remote-storage.md
index 0376e0a40..1d00a537d 100644
--- a/website/docs/maintenance/tiered-storage/remote-storage.md
+++ b/website/docs/maintenance/tiered-storage/remote-storage.md
@@ -6,7 +6,7 @@ sidebar_position: 2
# Remote Storage
Remote storage usually means a cost-efficient and fault-tolerant storage
comparing to local disk, such as S3, HDFS, OSS.
-See more detail about how to configure remote storage in documentation of
[filesystems](maintenance/filesystems/overview.md).
+See more detail about how to configure remote storage in documentation of
[filesystems](filesystems/overview.md).
For log table, Fluss will use remote storage to store the tiered log segments
of data. For primary key table, Fluss will use remote storage to store the
snapshot as well as the tiered log segments for change log.
diff --git a/website/docs/quickstart/flink.md b/website/docs/quickstart/flink.md
index 8ee3b7ba4..edb92924d 100644
--- a/website/docs/quickstart/flink.md
+++ b/website/docs/quickstart/flink.md
@@ -147,11 +147,11 @@ volumes:
The Docker Compose environment consists of the following containers:
- **RustFS:** an S3-compatible object storage for tiered storage. You can
access the RustFS console at http://localhost:9001 with credentials
`rustfsadmin/rustfsadmin`. An init container (`rustfs-init`) automatically
creates the `fluss` bucket on startup.
- **Fluss Cluster:** a Fluss `CoordinatorServer`, a Fluss `TabletServer` and a
`ZooKeeper` server.
- - Credentials are configured directly with `s3.access-key` and
`s3.secret-key`. The `s3.assumed.role.arn` and `s3.assumed.role.sts.endpoint`
options configure [AssumeRole
STS](/maintenance/filesystems/s3.md#assumerole-sts-configuration) which is
required by RustFS for delegation token support. Production systems should use
CredentialsProvider chain specific to cloud environments.
+ - Credentials are configured directly with `s3.access-key` and
`s3.secret-key`. The `s3.assumed.role.arn` and `s3.assumed.role.sts.endpoint`
options configure [AssumeRole
STS](../maintenance/tiered-storage/filesystems/s3.md#assumerole-sts-configuration)
which is required by RustFS for delegation token support. Production systems
should use CredentialsProvider chain specific to cloud environments.
- **Flink Cluster**: a Flink `JobManager`, a Flink `TaskManager`, and a Flink
SQL client container to execute queries. The
[`apache/fluss-quickstart-flink`](https://hub.docker.com/r/apache/fluss-quickstart-flink)
image bundles the Fluss Flink connector,
[flink-faker](https://github.com/knaufk/flink-faker) for demo data generation,
and S3 filesystem support, so no extra jar downloads are required for this
guide.
:::tip
-[RustFS](https://github.com/rustfs/rustfs) is used as replacement for S3 in
this quickstart example, for your production setup you may want to configure
this to use cloud file system. See [here](/maintenance/filesystems/overview.md)
for information on how to setup cloud file systems
+[RustFS](https://github.com/rustfs/rustfs) is used as replacement for S3 in
this quickstart example, for your production setup you may want to configure
this to use cloud file system. See
[here](/maintenance/tiered-storage/filesystems/overview.md) for information on
how to setup cloud file systems
:::
3. To start all containers, run:
diff --git a/website/docs/quickstart/lakehouse.md
b/website/docs/quickstart/lakehouse.md
index b9e560a92..3314a7267 100644
--- a/website/docs/quickstart/lakehouse.md
+++ b/website/docs/quickstart/lakehouse.md
@@ -182,7 +182,7 @@ The Docker Compose environment consists of the following
containers:
:::tip
-[RustFS](https://github.com/rustfs/rustfs) is used as replacement for S3 in
this quickstart example, for your production setup you may want to configure
this to use cloud file system. See [here](/maintenance/filesystems/overview.md)
for information on how to setup cloud file systems
+[RustFS](https://github.com/rustfs/rustfs) is used as replacement for S3 in
this quickstart example, for your production setup you may want to configure
this to use cloud file system. See
[here](/maintenance/tiered-storage/filesystems/overview.md) for information on
how to setup cloud file systems
:::
4. To start all containers, run:
@@ -396,7 +396,7 @@ The Docker Compose environment consists of the following
containers:
- **RustFS**: an S3-compatible storage system used both as Fluss remote
storage and Iceberg's filesystem warehouse.
:::tip
-[RustFS](https://github.com/rustfs/rustfs) is used as replacement for S3 in
this quickstart example, for your production setup you may want to configure
this to use cloud file system. See [here](/maintenance/filesystems/overview.md)
for information on how to setup cloud file systems
+[RustFS](https://github.com/rustfs/rustfs) is used as replacement for S3 in
this quickstart example, for your production setup you may want to configure
this to use cloud file system. See
[here](/maintenance/tiered-storage/filesystems/overview.md) for information on
how to setup cloud file systems
:::
4. To start all containers, run:
diff --git a/website/docs/streaming-lakehouse/datalake-catalogs/_category_.json
b/website/docs/streaming-lakehouse/datalake-catalogs/_category_.json
new file mode 100644
index 000000000..01d042de8
--- /dev/null
+++ b/website/docs/streaming-lakehouse/datalake-catalogs/_category_.json
@@ -0,0 +1,4 @@
+{
+ "label": "DataLake Catalogs",
+ "position": 5
+}
diff --git
a/website/docs/streaming-lakehouse/integrate-data-lakes/catalogs/lakekeeper.md
b/website/docs/streaming-lakehouse/datalake-catalogs/lakekeeper.md
similarity index 85%
rename from
website/docs/streaming-lakehouse/integrate-data-lakes/catalogs/lakekeeper.md
rename to website/docs/streaming-lakehouse/datalake-catalogs/lakekeeper.md
index b5fbf779c..c20462578 100644
---
a/website/docs/streaming-lakehouse/integrate-data-lakes/catalogs/lakekeeper.md
+++ b/website/docs/streaming-lakehouse/datalake-catalogs/lakekeeper.md
@@ -9,7 +9,7 @@ sidebar_position: 1
[Lakekeeper](https://lakekeeper.io/) is an Apache-licensed Iceberg REST
Catalog written in Rust. It provides a standards-compliant [Iceberg REST
Catalog](https://iceberg.apache.org/concepts/catalog/#decoupling-using-the-rest-catalog)
interface, making tiered Iceberg tables discoverable and queryable by any
Iceberg-compatible engine.
-This guide explains how to configure Fluss to use Lakekeeper as its Iceberg
catalog. For general Iceberg integration details (table mapping, data types,
limitations), see [Iceberg](../formats/iceberg.md).
+This guide explains how to configure Fluss to use Lakekeeper as its Iceberg
catalog. For general Iceberg integration details (table mapping, data types,
limitations), see [Iceberg](../datalake-formats/iceberg.md).
## Quick Start with Docker Compose
@@ -20,7 +20,7 @@ A complete Docker Compose example including Fluss, the
tiering service, and Lake
When Fluss is configured with Lakekeeper as its Iceberg REST catalog:
1. Fluss creates and manages Iceberg table metadata through Lakekeeper's REST
API
-2. The [tiering
service](maintenance/tiered-storage/lakehouse-storage.md#start-the-datalake-tiering-service)
writes data to object storage and commits snapshots via Lakekeeper
+2. The [tiering
service](../../install-deploy/deploying-streaming-lakehouse.md#starting-the-tiering-service)
writes data to object storage and commits snapshots via Lakekeeper
3. Any Iceberg-compatible engine (Flink, Spark, Trino, StarRocks, etc.) can
discover and query the tiered tables through Lakekeeper
## Prerequisites
@@ -91,11 +91,11 @@ wget -P ${FLUSS_HOME}/plugins/iceberg/ \
https://repo1.maven.org/maven2/io/trino/hadoop/hadoop-apache/3.3.5-2/hadoop-apache-3.3.5-2.jar
```
-See [Iceberg - Hadoop
Dependencies](../formats/iceberg.md#1-hadoop-dependencies-configuration) for
alternative approaches.
+See [Iceberg - Hadoop
Dependencies](../datalake-formats/iceberg.md#1-hadoop-dependencies-configuration)
for alternative approaches.
### Start Tiering Service
-Follow the [Iceberg tiering service
setup](../formats/iceberg.md#start-tiering-service-to-iceberg) to prepare the
required JARs and start the tiering service. Use REST catalog parameters when
launching the Flink tiering job:
+Follow the [Iceberg tiering service
setup](../datalake-formats/iceberg.md#start-tiering-service-to-iceberg) to
prepare the required JARs and start the tiering service. Use REST catalog
parameters when launching the Flink tiering job:
```bash
${FLINK_HOME}/bin/flink run /path/to/fluss-flink-tiering-$FLUSS_VERSION$.jar \
@@ -137,11 +137,11 @@ SET 'execution.runtime-mode' = 'batch';
SELECT COUNT(*) FROM orders;
```
-For details on union reads, streaming reads, and reading with other engines,
see [Iceberg - Read Tables](../formats/iceberg.md#read-tables).
+For details on union reads, streaming reads, and reading with other engines,
see [Iceberg - Read Tables](../datalake-formats/iceberg.md#read-tables).
## Further Reading
-- [Iceberg Integration](../formats/iceberg.md) - Table mapping, data types,
supported catalog types, and limitations
-- [Lakehouse Storage](maintenance/tiered-storage/lakehouse-storage.md) -
General tiered storage setup
+- [Iceberg Integration](../datalake-formats/iceberg.md) - Table mapping, data
types, supported catalog types, and limitations
+- [Lakehouse Storage](../../maintenance/tiered-storage/lakehouse-storage.md) -
General tiered storage setup
- [Lakekeeper Documentation](https://docs.lakekeeper.io) - Deploying and
managing Lakekeeper
- [Lakekeeper Fluss Engine
Docs](https://docs.lakekeeper.io/docs/nightly/engines/#apache-fluss) -
Lakekeeper-side configuration reference
diff --git
a/website/docs/streaming-lakehouse/integrate-data-lakes/catalogs/polaris.md
b/website/docs/streaming-lakehouse/datalake-catalogs/polaris.md
similarity index 90%
rename from
website/docs/streaming-lakehouse/integrate-data-lakes/catalogs/polaris.md
rename to website/docs/streaming-lakehouse/datalake-catalogs/polaris.md
index 82a242d36..3ce7bba95 100644
--- a/website/docs/streaming-lakehouse/integrate-data-lakes/catalogs/polaris.md
+++ b/website/docs/streaming-lakehouse/datalake-catalogs/polaris.md
@@ -9,14 +9,14 @@ sidebar_position: 2
[Apache Polaris](https://polaris.apache.org/) is an open-source,
fully-featured catalog for Apache Iceberg. It implements Iceberg's [REST
Catalog](https://iceberg.apache.org/concepts/catalog/#decoupling-using-the-rest-catalog)
interface, making Iceberg tables discoverable and queryable by any
Iceberg-compatible engine, with role-based access control and credential
vending built in.
-This guide explains how to configure Fluss to use Polaris as its Iceberg
catalog. For general Iceberg integration details (table mapping, data types,
limitations), see [Iceberg](../formats/iceberg.md).
+This guide explains how to configure Fluss to use Polaris as its Iceberg
catalog. For general Iceberg integration details (table mapping, data types,
limitations), see [Iceberg](../datalake-formats/iceberg.md).
## How It Works
When Fluss is configured with Polaris as its Iceberg REST catalog:
1. Fluss creates and manages Iceberg table metadata through Polaris's REST API
-2. The [tiering
service](maintenance/tiered-storage/lakehouse-storage.md#start-the-datalake-tiering-service)
writes data to object storage and commits snapshots via Polaris
+2. The [tiering
service](../../install-deploy/deploying-streaming-lakehouse.md#starting-the-tiering-service)
writes data to object storage and commits snapshots via Polaris
3. Any Iceberg-compatible engine (Flink, Spark, Trino, StarRocks, etc.) can
discover and query the tiered tables through Polaris
## Prerequisites
@@ -90,7 +90,7 @@ Fluss strips the `datalake.iceberg.` prefix and passes the
remaining properties
### Start Tiering Service
-Follow the [Iceberg tiering service
setup](../formats/iceberg.md#start-tiering-service-to-iceberg) to prepare the
required JARs and start the tiering service. Use the REST catalog parameters
when launching the Flink tiering job:
+Follow the [Iceberg tiering service
setup](../datalake-formats/iceberg.md#start-tiering-service-to-iceberg) to
prepare the required JARs and start the tiering service. Use the REST catalog
parameters when launching the Flink tiering job:
```bash
${FLINK_HOME}/bin/flink run /path/to/fluss-flink-tiering-$FLUSS_VERSION$.jar \
@@ -137,10 +137,10 @@ SET 'execution.runtime-mode' = 'batch';
SELECT COUNT(*) FROM orders;
```
-For details on union reads, streaming reads, and reading with other engines,
see [Iceberg - Read Tables](../formats/iceberg.md#read-tables).
+For details on union reads, streaming reads, and reading with other engines,
see [Iceberg - Read Tables](../datalake-formats/iceberg.md#read-tables).
## Further Reading
-- [Iceberg Integration](../formats/iceberg.md) - Table mapping, data types,
supported catalog types, and limitations
-- [Lakehouse Storage](maintenance/tiered-storage/lakehouse-storage.md) -
General tiered storage setup
+- [Iceberg Integration](../datalake-formats/iceberg.md) - Table mapping, data
types, supported catalog types, and limitations
+- [Lakehouse Storage](../../maintenance/tiered-storage/lakehouse-storage.md) -
General tiered storage setup
- [Apache Polaris Documentation](https://polaris.apache.org/) - Deploying and
managing Polaris, catalogs, principals, and access control
diff --git a/website/docs/streaming-lakehouse/datalake-formats/_category_.json
b/website/docs/streaming-lakehouse/datalake-formats/_category_.json
new file mode 100644
index 000000000..b42be6924
--- /dev/null
+++ b/website/docs/streaming-lakehouse/datalake-formats/_category_.json
@@ -0,0 +1,4 @@
+{
+ "label": "DataLake Formats",
+ "position": 4
+}
diff --git
a/website/docs/streaming-lakehouse/integrate-data-lakes/formats/iceberg.md
b/website/docs/streaming-lakehouse/datalake-formats/iceberg.md
similarity index 88%
rename from
website/docs/streaming-lakehouse/integrate-data-lakes/formats/iceberg.md
rename to website/docs/streaming-lakehouse/datalake-formats/iceberg.md
index bb54356c1..48f1eb93d 100644
--- a/website/docs/streaming-lakehouse/integrate-data-lakes/formats/iceberg.md
+++ b/website/docs/streaming-lakehouse/datalake-formats/iceberg.md
@@ -8,7 +8,7 @@ sidebar_position: 2
## Introduction
[Apache Iceberg](https://iceberg.apache.org/) is an open table format for huge
analytic datasets. It provides ACID transactions, schema evolution, and
efficient data organization for data lakes.
-To integrate Fluss with Iceberg, you must enable lakehouse storage and
configure Iceberg as the lakehouse storage. For more details, see [Enable
Lakehouse
Storage](maintenance/tiered-storage/lakehouse-storage.md#enable-lakehouse-storage).
+To integrate Fluss with Iceberg, you must enable lakehouse storage and
configure Iceberg as the lakehouse storage. For more details, see [Deploying
Streaming Lakehouse](../../install-deploy/deploying-streaming-lakehouse.md).
> **NOTE**: Iceberg requires JDK11 or later. Please ensure that both your
> Fluss deployment and the Flink cluster used for tiering services are running
> on JDK11+.
@@ -123,7 +123,7 @@ The Iceberg version that Fluss bundles is based on
`1.10.1`. Please ensure the J
### Start Tiering Service to Iceberg
-To tier Fluss's data to Iceberg, you must start the datalake tiering service.
For guidance, you can refer to [Start The Datalake Tiering
Service](maintenance/tiered-storage/lakehouse-storage.md#start-the-datalake-tiering-service).
Although the example uses Paimon, the process is also applicable to Iceberg.
+To tier Fluss's data to Iceberg, you must start the datalake tiering service.
For guidance, you can refer to [Deploying Streaming
Lakehouse](../../install-deploy/deploying-streaming-lakehouse.md). Although the
example uses Paimon, the process is also applicable to Iceberg.
#### Prerequisites: Hadoop Dependencies
@@ -166,17 +166,17 @@ export HADOOP_CLASSPATH=`$HADOOP_HOME/bin/hadoop
classpath`
#### Prepare Required JARs
-Follow the dependency management guidelines below for the [Prepare required
jars](maintenance/tiered-storage/lakehouse-storage.md#prepare-required-jars)
step:
+Follow the dependency management guidelines below for required JARs:
##### 1. Core Fluss Components
-- **Fluss Flink Connector**: Put the Fluss Flink connector JAR into
`${FLINK_HOME}/lib` — see
[Dependencies](../../../engine-flink/getting-started.md#dependencies) for all
supported Flink versions.
+- **Fluss Flink Connector**: Put the Fluss Flink connector JAR into
`${FLINK_HOME}/lib` — see
[Dependencies](../../engine-flink/getting-started.md#dependencies) for all
supported Flink versions.
- For Flink 1.20:
[fluss-flink-1.20-$FLUSS_VERSION$.jar]($FLUSS_MAVEN_REPO_URL$/org/apache/fluss/fluss-flink-1.20/$FLUSS_VERSION$/fluss-flink-1.20-$FLUSS_VERSION$.jar)
##### 2. Remote Storage Support
If you are using remote storage, download the corresponding Fluss filesystem
JAR and place it into `${FLINK_HOME}/lib`:
-- **Amazon S3**: see [S3
Dependencies](../../../maintenance/filesystems/s3.md#dependencies)
-- **Aliyun OSS**: see [OSS
Dependencies](../../../maintenance/filesystems/oss.md#dependencies)
-- **HDFS**: see [HDFS
Dependencies](../../../maintenance/filesystems/hdfs.md#dependencies)
+- **Amazon S3**: see [S3
Dependencies](../../maintenance/tiered-storage/filesystems/s3.md#dependencies)
+- **Aliyun OSS**: see [OSS
Dependencies](../../maintenance/tiered-storage/filesystems/oss.md#dependencies)
+- **HDFS**: see [HDFS
Dependencies](../../maintenance/tiered-storage/filesystems/hdfs.md#dependencies)
##### 3. Iceberg Lake Connector
- **Fluss Lake Iceberg**: Put
[fluss-lake-iceberg-$FLUSS_VERSION$.jar]($FLUSS_MAVEN_REPO_URL$/org/apache/fluss/fluss-lake-iceberg/$FLUSS_VERSION$/fluss-lake-iceberg-$FLUSS_VERSION$.jar)
into `${FLINK_HOME}/lib`
@@ -197,7 +197,7 @@ failsafe-3.3.2.jar
#### Start Datalake Tiering Service
-When following the [Start Datalake Tiering
Service](maintenance/tiered-storage/lakehouse-storage.md#start-datalake-tiering-service)
guide, use Iceberg-specific configurations as parameters when starting the
Flink tiering job:
+Use Iceberg-specific configurations as parameters when starting the Flink
tiering job:
```bash
<FLINK_HOME>/bin/flink run /path/to/fluss-flink-tiering-$FLUSS_VERSION$.jar \
@@ -262,7 +262,7 @@ CREATE TABLE fluss_order_with_lake (
) WITH (
'table.datalake.enabled' = 'true',
'table.datalake.freshness' = '30s',
- 'table.datalake.auto-maintenance' = 'true',
+ 'table.datalake.auto-compaction' = 'true',
'iceberg.write.format.default' = 'orc',
'iceberg.commit.retry.num-retries' = '5'
);
@@ -441,28 +441,56 @@ You need to place the JARs required by Iceberg to read
data into `${FLINK_HOME}/
##### Union Read
-To read the full dataset, which includes both Fluss (fresh) and Iceberg
(historical) data, simply query the table without any suffix. The following
example illustrates this:
+To read the full dataset, which includes both Fluss (fresh) and Iceberg
(historical) data, simply query the table without any suffix.
-```sql
--- Set execution mode to streaming or batch, here just take batch as an example
+**Table type support:**
+- **Log tables**: Supported in both batch and streaming mode.
+- **Primary key tables**: Supported in both batch and streaming mode. Updates
tiered to Iceberg are written as equality deletes.
+
+```sql title="Batch mode"
SET 'execution.runtime-mode' = 'batch';
-- Query will union data from Fluss and Iceberg
-select SUM(visit_count) from fluss_access_log;
-```
+SELECT SUM(visit_count) FROM fluss_access_log;
-It supports both batch and streaming modes, utilizing Iceberg for historical
data and Fluss for fresh data:
+-- Aggregations work across both data sources
+SELECT COUNT(*), SUM(visit_count) FROM fluss_access_log;
-- **Batch mode** (only log table)
+-- Predicates are applied across both Fluss and Iceberg data
+SELECT * FROM fluss_access_log WHERE visit_date >= '2024-01-01';
+```
-- **Streaming mode** (primary key table and log table)
+```sql title="Streaming mode"
+SET 'execution.runtime-mode' = 'streaming';
- Flink first reads the latest Iceberg snapshot (tiered via tiering service),
then switches to Fluss starting from the log offset matching that snapshot.
This design minimizes Fluss storage requirements (reducing costs) while using
Iceberg as a complete historical archive.
+-- Reads the latest Iceberg snapshot first, then continues from Fluss
+SELECT * FROM fluss_access_log;
+```
+
+Flink first reads the latest Iceberg snapshot (tiered via tiering service),
then switches to Fluss starting from the log offset matching that snapshot.
This design minimizes Fluss storage requirements (reducing costs) while using
Iceberg as a complete historical archive.
Key behavior for data retention:
- **Expired Fluss log data** (controlled by `table.log.ttl`) remains
accessible via Iceberg if previously tiered
- **Cleaned-up partitions** in partitioned tables (controlled by
`table.auto-partition.num-retention`) remain accessible via Iceberg if
previously tiered
+#### Lake-Only Read
+
+To query only the data already tiered to Iceberg, add the `$lake` suffix to
the table name:
+
+```sql title="Lake-only read"
+-- Reads only the data tiered to Iceberg, not the fresh data still in Fluss
+SELECT * FROM fluss_access_log$lake;
+```
+
+Data becomes visible through `$lake` as soon as the tiering service commits
it: each tiering round produces one Iceberg snapshot, and the commit is
readable immediately once it succeeds — there's no separate flush or refresh
step.
+
+You can also query Iceberg's system tables through the `$lake$<system-table>`
suffix, for example to inspect commit history via the `snapshots` table:
+
+```sql title="Read Iceberg snapshots metadata"
+-- Each row corresponds to one tiering commit
+SELECT committed_at, snapshot_id, operation, summary FROM
fluss_access_log$lake$snapshots;
+```
+
### Reading with Other Engines
Since data tiered to Iceberg from Fluss is stored as standard Iceberg tables,
you can use any Iceberg-compatible engine. Below is an example using
[StarRocks](https://docs.starrocks.io/docs/data_source/catalog/iceberg/iceberg_catalog/):
diff --git
a/website/docs/streaming-lakehouse/integrate-data-lakes/formats/lance.md
b/website/docs/streaming-lakehouse/datalake-formats/lance.md
similarity index 89%
rename from
website/docs/streaming-lakehouse/integrate-data-lakes/formats/lance.md
rename to website/docs/streaming-lakehouse/datalake-formats/lance.md
index 85f159dc1..0b016539e 100644
--- a/website/docs/streaming-lakehouse/integrate-data-lakes/formats/lance.md
+++ b/website/docs/streaming-lakehouse/datalake-formats/lance.md
@@ -8,7 +8,7 @@ sidebar_position: 3
## Introduction
[Lance](https://lancedb.github.io/lance/) is a modern table format optimized
for machine learning and AI applications.
-To integrate Fluss with Lance, you must enable lakehouse storage and configure
Lance as the lakehouse storage. For more details, see [Enable Lakehouse
Storage](maintenance/tiered-storage/lakehouse-storage.md#enable-lakehouse-storage).
+To integrate Fluss with Lance, you must enable lakehouse storage and configure
Lance as the lakehouse storage. For more details, see [Deploying Streaming
Lakehouse](../../install-deploy/deploying-streaming-lakehouse.md).
## Dependencies
@@ -69,16 +69,15 @@ CREATE TABLE fluss_order_with_lake (
```
### Start Tiering Service to Lance
-Then, you must start the datalake tiering service to tier Fluss's data to
Lance. For guidance, you can refer to [Start The Datalake Tiering Service
-](maintenance/tiered-storage/lakehouse-storage.md#start-the-datalake-tiering-service).
Although the example uses Paimon, the process is also applicable to Lance.
+Then, you must start the datalake tiering service to tier Fluss's data to
Lance. For guidance, you can refer to [Deploying Streaming
Lakehouse](../../install-deploy/deploying-streaming-lakehouse.md). Although the
example uses Paimon, the process is also applicable to Lance.
-But in [Prepare required
jars](maintenance/tiered-storage/lakehouse-storage.md#prepare-required-jars)
step, you should follow this guidance:
-- Put the Fluss Flink connector JAR into `${FLINK_HOME}/lib`; pick the
connector matching your Flink version (see
[Dependencies](../../../engine-flink/getting-started.md#dependencies)). For
Flink 1.20, use
[fluss-flink-1.20-$FLUSS_VERSION$.jar]($FLUSS_MAVEN_REPO_URL$/org/apache/fluss/fluss-flink-1.20/$FLUSS_VERSION$/fluss-flink-1.20-$FLUSS_VERSION$.jar).
-- If you are using [Amazon S3](http://aws.amazon.com/s3/), [Aliyun
OSS](https://www.aliyun.com/product/oss) or [HDFS(Hadoop Distributed File
System)](https://hadoop.apache.org/docs/stable/) as Fluss's [remote
storage](maintenance/tiered-storage/remote-storage.md),
- you should download the corresponding Fluss filesystem JAR (see the
[Filesystems](../../../maintenance/filesystems/overview.md) section) and also
put it into `${FLINK_HOME}/lib`.
+For required JARs, you should follow this guidance:
+- Put the Fluss Flink connector JAR into `${FLINK_HOME}/lib`; pick the
connector matching your Flink version (see
[Dependencies](../../engine-flink/getting-started.md#dependencies)). For Flink
1.20, use
[fluss-flink-1.20-$FLUSS_VERSION$.jar]($FLUSS_MAVEN_REPO_URL$/org/apache/fluss/fluss-flink-1.20/$FLUSS_VERSION$/fluss-flink-1.20-$FLUSS_VERSION$.jar).
+- If you are using [Amazon S3](http://aws.amazon.com/s3/), [Aliyun
OSS](https://www.aliyun.com/product/oss) or [HDFS(Hadoop Distributed File
System)](https://hadoop.apache.org/docs/stable/) as Fluss's [remote
storage](../../maintenance/tiered-storage/remote-storage.md),
+ you should download the corresponding Fluss filesystem JAR (see
[Filesystems](../../maintenance/tiered-storage/filesystems/overview.md)) and
also put it into `${FLINK_HOME}/lib`.
- Put
[fluss-lake-lance-$FLUSS_VERSION$.jar]($FLUSS_MAVEN_REPO_URL$/org/apache/fluss/fluss-lake-lance/$FLUSS_VERSION$/fluss-lake-lance-$FLUSS_VERSION$.jar)
into `${FLINK_HOME}/lib`.
-Additionally, when following the [Start Datalake Tiering
Service](maintenance/tiered-storage/lakehouse-storage.md#start-datalake-tiering-service)
guide, make sure to use Lance-specific configurations as parameters when
starting the Flink tiering job:
+Make sure to use Lance-specific configurations as parameters when starting the
Flink tiering job:
```shell
<FLINK_HOME>/bin/flink run /path/to/fluss-flink-tiering-$FLUSS_VERSION$.jar \
--fluss.bootstrap.servers localhost:9123 \
@@ -126,7 +125,7 @@ ds =
lance.dataset("<warehouse_path>/<database_name>/<table_name>.lance")
Lance internally stores data in Arrow format.
When integrating with Lance, Fluss automatically converts between Fluss data
types and Lance data types.
-The following table shows the mapping between [Fluss data
types](table-design/data-types.md) and Lance data types:
+The following table shows the mapping between [Fluss data
types](../../table-design/data-types.md) and Lance data types:
| Fluss Data Type | Lance Data Type |
|-------------------------------|---------------------------|
diff --git
a/website/docs/streaming-lakehouse/integrate-data-lakes/formats/paimon.md
b/website/docs/streaming-lakehouse/datalake-formats/paimon.md
similarity index 95%
rename from
website/docs/streaming-lakehouse/integrate-data-lakes/formats/paimon.md
rename to website/docs/streaming-lakehouse/datalake-formats/paimon.md
index 595c5cb28..4a9e42ef7 100644
--- a/website/docs/streaming-lakehouse/integrate-data-lakes/formats/paimon.md
+++ b/website/docs/streaming-lakehouse/datalake-formats/paimon.md
@@ -8,7 +8,7 @@ sidebar_position: 1
## Introduction
[Apache Paimon](https://paimon.apache.org/) innovatively combines a lake
format with an LSM (Log-Structured Merge-tree) structure, bringing efficient
updates into the lake architecture.
-To integrate Fluss with Paimon, you must enable lakehouse storage and
configure Paimon as the lakehouse storage. For more details, see [Enable
Lakehouse
Storage](maintenance/tiered-storage/lakehouse-storage.md#enable-lakehouse-storage).
+To integrate Fluss with Paimon, you must enable lakehouse storage and
configure Paimon as the lakehouse storage. For more details, see [Deploying
Streaming Lakehouse](../../install-deploy/deploying-streaming-lakehouse.md).
## Dependencies
@@ -39,7 +39,7 @@ Verify downloaded JARs against the [KEYS
file](https://downloads.apache.org/incu
## Configure Paimon as LakeHouse Storage
-For general guidance on configuring Paimon as the lakehouse storage, you can
refer to [Lakehouse Storage](maintenance/tiered-storage/lakehouse-storage.md)
documentation. When starting the tiering service, make sure to use
Paimon-specific configurations as parameters.
+For general guidance on configuring Paimon as the lakehouse storage, you can
refer to [Deploying Streaming
Lakehouse](../../install-deploy/deploying-streaming-lakehouse.md)
documentation. When starting the tiering service, make sure to use
Paimon-specific configurations as parameters.
When a table is created or altered with the option `'table.datalake.enabled' =
'true'`, Fluss will automatically create a corresponding Paimon table with the
same table path.
The schema of the Paimon table matches that of the Fluss table, except for the
addition of three system columns at the end: `__bucket`, `__offset`, and
`__timestamp`.
@@ -147,8 +147,8 @@ It supports both batch and streaming modes, using Paimon
for historical data and
This design enables Fluss to store only a small portion of the dataset in
the Fluss cluster, reducing costs, while Paimon serves as the source of
complete historical data when needed.
Key behavior for data retention:
-- **Expired Fluss log data** (controlled by `table.log.ttl`) remains
accessible via Iceberg if previously tiered
-- **Cleaned-up partitions** in partitioned tables (controlled by
`table.auto-partition.num-retention`) remain accessible via Iceberg if
previously tiered
+- **Expired Fluss log data** (controlled by `table.log.ttl`) remains
accessible via Paimon if previously tiered
+- **Cleaned-up partitions** in partitioned tables (controlled by
`table.auto-partition.num-retention`) remain accessible via Paimon if
previously tiered
### Reading with other Engines
@@ -182,7 +182,7 @@ SELECT * FROM
paimon_catalog.fluss.enriched_orders$snapshots;
## Data Type Mapping
When integrating with Paimon, Fluss automatically converts between Fluss data
types and Paimon data types.
-The following table shows the mapping between [Fluss data
types](table-design/data-types.md) and Paimon data types:
+The following table shows the mapping between [Fluss data
types](../../table-design/data-types.md) and Paimon data types:
| Fluss Data Type | Paimon
Data Type |
|-----------------------------------------------------------------|-----------------------------------------------------------------|
@@ -253,4 +253,4 @@ For partitioned tables, the metadata structure includes
partition information:
| `partition_id` | Unique identifier in Fluss for the partition | `0`, `1`
|
| `bucket` | Bucket identifier within the partition | `0`, `1`,
`2` |
| `partition_name` | Human-readable partition name |
`"date=2025"`, `"date=2026"` |
-| `offset` | Offset within the partition's log | `3`,
`1000` |
\ No newline at end of file
+| `offset` | Offset within the partition's log | `3`,
`1000` |
diff --git
a/website/docs/streaming-lakehouse/integrate-data-lakes/_category_.json
b/website/docs/streaming-lakehouse/integrate-data-lakes/_category_.json
deleted file mode 100644
index 095e56dff..000000000
--- a/website/docs/streaming-lakehouse/integrate-data-lakes/_category_.json
+++ /dev/null
@@ -1,4 +0,0 @@
-{
- "label": "Integrate Data Lakes",
- "position": 2
-}
diff --git
a/website/docs/streaming-lakehouse/integrate-data-lakes/catalogs/_category_.json
b/website/docs/streaming-lakehouse/integrate-data-lakes/catalogs/_category_.json
deleted file mode 100644
index 0586a8325..000000000
---
a/website/docs/streaming-lakehouse/integrate-data-lakes/catalogs/_category_.json
+++ /dev/null
@@ -1,4 +0,0 @@
-{
- "label": "Data Lake Catalogs",
- "position": 2
-}
diff --git
a/website/docs/streaming-lakehouse/integrate-data-lakes/formats/_category_.json
b/website/docs/streaming-lakehouse/integrate-data-lakes/formats/_category_.json
deleted file mode 100644
index b6c3a5fdc..000000000
---
a/website/docs/streaming-lakehouse/integrate-data-lakes/formats/_category_.json
+++ /dev/null
@@ -1,4 +0,0 @@
-{
- "label": "Data Lake Formats",
- "position": 1
-}
diff --git a/website/docs/streaming-lakehouse/overview.md
b/website/docs/streaming-lakehouse/overview.md
index 476ec6813..d86ab3e57 100644
--- a/website/docs/streaming-lakehouse/overview.md
+++ b/website/docs/streaming-lakehouse/overview.md
@@ -25,7 +25,7 @@ Overall, these data lake formats typically achieve data
freshness at best within
## Streaming Lakehouse: Unified Stream & Lakehouse
Fluss is a streaming storage supporting streaming reads and writes with
sub-second low latency.
-With [Lakehouse Storage](maintenance/tiered-storage/lakehouse-storage.md),
Fluss unifies data streaming and data Lakehouse by serving real-time streaming
data on top of Lakehouse.
+With [Lakehouse Storage](../maintenance/tiered-storage/lakehouse-storage.md),
Fluss unifies data streaming and data Lakehouse by serving real-time streaming
data on top of Lakehouse.
This not only brings low latency to data Lakehouse, but also adds powerful
analytics to data streams.
To build a Streaming Lakehouse, Fluss maintains a tiering service that
compacts real-time data from the Fluss cluster into the data lake format stored
in the Lakehouse Storage.
@@ -43,4 +43,13 @@ Some powerful features it provides are:
- **Analytical Streams**: The union reads help data streams to have the
powerful analytics capabilities. This reduces complexity when developing
streaming applications, simplifies debugging, and allows for immediate access
to live data insights.
- **Connect to Lakehouse Ecosystem**: Fluss keeps the table metadata in sync
with data lake catalogs while compacting data into Lakehouse. As a result,
external engines like Spark, StarRocks, Flink, and Trino can read the data
directly. They simply connect to the data lake catalog.
-Currently, Fluss supports
[Paimon](streaming-lakehouse/integrate-data-lakes/formats/paimon.md),
[Iceberg](streaming-lakehouse/integrate-data-lakes/formats/iceberg.md), and
[Lance](streaming-lakehouse/integrate-data-lakes/formats/lance.md) as Lakehouse
Storage, more kinds of data lake formats are on the roadmap.
+Currently, Fluss supports [Paimon](datalake-formats/paimon.md),
[Iceberg](datalake-formats/iceberg.md), and [Lance](datalake-formats/lance.md)
as Lakehouse Storage, more kinds of data lake formats are on the roadmap.
+
+## Getting Started
+
+Ready to build your Streaming Lakehouse? Here's a recommended path:
+
+1. **Hands-on Tutorial**: Follow the [Lakehouse
Quickstart](../quickstart/lakehouse.md) to build a working Streaming Lakehouse
in minutes using Docker
+2. **Understand the Architecture**: Learn how [Tiering
Service](tiering-service.md) continuously syncs data to your data lake
+3. **Query Your Data**: Explore [Union Read](union-read.md) to query real-time
and historical data together
+4. **Production Deployment**: See [Deploying Streaming
Lakehouse](../install-deploy/deploying-streaming-lakehouse.md) for cluster setup
diff --git a/website/docs/streaming-lakehouse/tiering-service.md
b/website/docs/streaming-lakehouse/tiering-service.md
new file mode 100644
index 000000000..f92815931
--- /dev/null
+++ b/website/docs/streaming-lakehouse/tiering-service.md
@@ -0,0 +1,102 @@
+---
+title: "Tiering Service"
+sidebar_position: 2
+---
+
+# Tiering Service
+
+The Tiering Service continuously compacts real-time data from Fluss into the
configured lake format (Paimon, Iceberg, or Lance) for cost-efficient long-term
storage and analytics.
+
+## Overview
+
+The Tiering Service is implemented as an Apache Flink job that:
+- Reads records from Fluss tables with lakehouse storage enabled
+- Writes data to the configured data lake format (Paimon, Iceberg, or Lance)
+- Maintains exactly-once semantics between Fluss and the data lake
+- Operates incrementally, syncing only missing data segments
+
+For deployment instructions, see [Deploying Streaming
Lakehouse](../install-deploy/deploying-streaming-lakehouse.md).
+
+## Architecture
+
+The Tiering Service consists of three Flink operators:
+
+| Operator | Description |
+|----------|-------------|
+| **TieringSource** | Reads records from Fluss and writes to the data lake via
LakeWriter |
+| **TieringCommitter** | Commits batches and advances offsets in both Fluss
and lake |
+| **No-Op Sink** | Dummy sink required by Flink's topology |
+
+### How It Works
+
+1. **Enumerator** sends heartbeat to Fluss CoordinatorService, receives table
metadata
+2. **SplitGenerator** calculates data delta between Fluss offsets and lake
snapshot offsets
+3. **Splits** are created for missing data ranges and assigned to readers
+4. **Readers** fetch records from Fluss tablet servers, write via LakeWriter
+5. **Committer** performs two-phase commit: first to lake, then updates Fluss
coordinator
+6. Cycle repeats based on `table.datalake.freshness` interval
+
+### Split Types
+
+| Table Type | Split Type | Description |
+|------------|------------|-------------|
+| Log Table (append-only) | TieringLogSplit | Defines starting and stopping
offsets for a contiguous range |
+| Primary Key Table | TieringSnapshotSplit | References snapshot ID and log
offset for CDC replay |
+
+## Configuration Options
+
+### Tiering Service Options
+
+The tiering job is a standalone Flink job, and its arguments fall into three
groups:
+
+- `--fluss.*` — Fluss client configuration, e.g. `--fluss.bootstrap.servers
localhost:9123`
+- `--datalake.<format>.*` — lake catalog/storage configuration; Fluss strips
the `datalake.<format>.` prefix before passing the remaining keys to the lake
connector
+- `--lake.tiering.*` — tiering job-level configuration, e.g.
`--lake.tiering.auto-expire-snapshot true`
+
+For example:
+
+```shell
+${FLINK_HOME}/bin/flink run \
+ -Dparallelism.default=3 \
+ /path/to/fluss-flink-tiering-$FLUSS_VERSION$.jar \
+ --fluss.bootstrap.servers localhost:9123 \
+ --datalake.format paimon \
+ --datalake.paimon.metastore filesystem \
+ --datalake.paimon.warehouse /tmp/paimon \
+ --lake.tiering.auto-expire-snapshot true
+```
+
+The following `--lake.tiering.*` options are set when starting the tiering job:
+
+| Option | Type | Default | Description |
+|--------|------|---------|-------------|
+| `lake.tiering.auto-expire-snapshot` | Boolean | false | Auto-trigger
snapshot expiration on commit |
+
+### Table-Level Options
+
+The following `table.datalake.*` options are configured per table when
creating or altering tables, not on the tiering job:
+
+| Option | Type | Default | Description |
+|--------|------|---------|-------------|
+| `table.datalake.enabled` | Boolean | false | Enable lakehouse storage for
this table |
+| `table.datalake.freshness` | Duration | 3min | Maximum lag between Fluss and
lake data |
+| `table.datalake.auto-compaction` | Boolean | false | Auto-trigger compaction
in the data lake |
+| `table.datalake.auto-expire-snapshot` | Boolean | false | Auto-expire
snapshots in the data lake |
+
+## Scaling
+
+The Tiering Service is stateless and can be scaled by adjusting Flink
parallelism:
+
+```shell
+${FLINK_HOME}/bin/flink run \
+ -Dparallelism.default=6 \
+ /path/to/fluss-flink-tiering-$FLUSS_VERSION$.jar \
+ --fluss.bootstrap.servers localhost:9123 \
+ ...
+```
+
+Multiple tiering service jobs can run simultaneously. They are coordinated by
the Fluss cluster to ensure exactly-once semantics and automatic load balancing.
+
+## Monitoring
+
+Key metrics for monitoring the Tiering Service are available through Flink's
metrics system. See [Monitoring
Metrics](../maintenance/observability/monitor-metrics.md#tiering-service-metrics)
for details on lakehouse tiering metrics.
diff --git a/website/docs/streaming-lakehouse/union-read.md
b/website/docs/streaming-lakehouse/union-read.md
new file mode 100644
index 000000000..e0e901f8f
--- /dev/null
+++ b/website/docs/streaming-lakehouse/union-read.md
@@ -0,0 +1,113 @@
+---
+title: "Union Read"
+sidebar_position: 3
+---
+
+# Union Read
+
+Union Read is a core feature of Fluss's Streaming Lakehouse that combines
real-time data from Fluss with historical data from the data lake in a single
query.
+
+## Overview
+
+For a table with `'table.datalake.enabled' = 'true'`, data exists in two
layers:
+- **Fluss (hot data)**: Sub-second fresh data stored in Arrow format
+- **Data Lake (cold data)**: Historical data stored in the configured lake
format
+
+Union Read transparently merges data from both sources, providing sub-second
freshness with full historical coverage.
+
+## Querying Tables
+
+### Union Read (Default)
+
+Query the table directly to read combined Fluss and lake data:
+
+```sql title="Flink SQL"
+-- Union Read: combines real-time Fluss data + historical lake data
+SELECT * FROM my_table;
+
+-- Aggregations work across both data sources
+SELECT COUNT(*), SUM(amount) FROM orders;
+```
+
+### Lake-Only Read
+
+To query only the data stored in the data lake, use the `$lake` suffix:
+
+```sql title="Flink SQL"
+-- Lake-only read: queries only tiered data
+SELECT * FROM my_table$lake;
+
+-- Access lake-specific system tables
+SELECT snapshot_id, total_record_count FROM my_table$lake$snapshots;
+```
+
+Lake-only queries are useful when:
+- Real-time freshness is not required
+- You need to access lake format-specific system tables
+- You want optimized performance for large historical scans
+
+## Execution Modes
+
+Union Read supports both batch and streaming modes:
+
+### Batch Mode
+
+```sql title="Flink SQL"
+SET 'execution.runtime-mode' = 'batch';
+SELECT SUM(total_price) FROM orders;
+```
+
+The query merges rows from both lake and Fluss, returning the most up-to-date
results. Multiple executions may produce different outputs as data is
continuously ingested.
+
+### Streaming Mode
+
+```sql title="Flink SQL"
+SET 'execution.runtime-mode' = 'streaming';
+SELECT * FROM orders;
+```
+
+Flink first reads the latest lake snapshot, then switches to Fluss starting
from the log offset aligned with that snapshot, ensuring exactly-once semantics.
+
+## Data Deduplication
+
+For **primary key tables**, Union Read automatically deduplicates records with
the same key, keeping the latest version from Fluss if it exists.
+
+For **log tables** (append-only), Union Read concatenates data from both
sources without deduplication.
+
+## Data Freshness
+
+The `table.datalake.freshness` option controls how often data is tiered to the
lake:
+
+```sql title="Flink SQL"
+CREATE TABLE orders (
+ order_id BIGINT PRIMARY KEY NOT ENFORCED,
+ amount DECIMAL(15, 2)
+) WITH (
+ 'table.datalake.enabled' = 'true',
+ 'table.datalake.freshness' = '1min'
+);
+```
+
+- **Shorter freshness** (e.g., `30s`): Lake data stays closer to real-time,
less data read from Fluss
+- **Longer freshness** (e.g., `10min`): Better lake read performance due to
larger files, more data read from Fluss
+
+## Data Retention
+
+Key behavior for data retention with Union Read:
+- **Expired Fluss log data** (controlled by `table.log.ttl`) remains
accessible via the lake if previously tiered
+- **Cleaned-up partitions** in partitioned tables (controlled by
`table.auto-partition.num-retention`) remain accessible via the lake if
previously tiered
+
+## Engine Support
+
+| Engine | Union Read | Lake-Only Read |
+|--------|------------|----------------|
+| Apache Flink | ✅ | ✅ Via `$lake` suffix |
+| Apache Spark | ✅ | ✅ Via native lake connectors |
+| Trino | ❌ | ✅ Via native lake connectors |
+| StarRocks | ❌ | ✅ Via native lake connectors |
+
+For Spark union read usage, see [Spark -
Reads](../engine-spark/reads.md#lake-enabled-tables-union-read).
+
+External engines can access the tiered lake data directly through native lake
format connectors. See the specific data lake format documentation for examples:
+- [Paimon - Reading with other
Engines](datalake-formats/paimon.md#reading-with-other-engines)
+- [Iceberg - Reading with other
Engines](datalake-formats/iceberg.md#reading-with-other-engines)
diff --git a/website/docs/table-design/table-types/log-table.md
b/website/docs/table-design/table-types/log-table.md
index dca2f78dd..405aa272e 100644
--- a/website/docs/table-design/table-types/log-table.md
+++ b/website/docs/table-design/table-types/log-table.md
@@ -145,4 +145,4 @@ SELECT * FROM log_table$changelog;
For detailed information, see the [Virtual
Tables](/table-design/virtual-tables.md) documentation.
## Log Tiering
-Log Table supports tiering data to different storage tiers. See more details
about [Remote Log](maintenance/tiered-storage/remote-storage.md).
+Log Table supports tiering data to different storage tiers. See more details
about [Remote Log](../../maintenance/tiered-storage/remote-storage.md).
diff --git a/website/docusaurus.config.ts b/website/docusaurus.config.ts
index 1e8b4e81a..1ab7eeb92 100644
--- a/website/docusaurus.config.ts
+++ b/website/docusaurus.config.ts
@@ -197,19 +197,38 @@ const config: Config = {
if (!existingPath.startsWith('/docs/')) {
return undefined;
}
-
+
// Extract the relative path after /docs/
- const relativeDocsPath = existingPath.substring(6);
+ const relativeDocsPath = existingPath.substring(6);
const firstSegment = relativeDocsPath.split('/')[0];
-
+
// Exclude any known version identifiers aligned with existing
routes
const existingVersionedRoutes = ['next',
...Object.keys(versionsMap)];
if (existingVersionedRoutes.includes(firstSegment)) {
return undefined;
}
-
- // Redirect the explicit versioned path to the implicit
unversioned path
- return [`/docs/${latestVersion}${existingPath.replace('/docs',
'')}`];
+
+ const redirects = [
+ // Redirect the explicit versioned path to the implicit
unversioned path
+ `/docs/${latestVersion}${existingPath.replace('/docs', '')}`,
+ ];
+
+ // Preserve the previously published URLs for docs pages that were
+ // moved/renamed. These are keyed off the new (existing) route, so
the
+ // redirect target is always valid, and the old URL is preserved
once
+ // the restructured version becomes the latest unversioned release.
+ const renameRules = [
+ { from: '/maintenance/filesystems/', to:
'/maintenance/tiered-storage/filesystems/' },
+ { from: '/streaming-lakehouse/integrate-data-lakes/formats/',
to: '/streaming-lakehouse/datalake-formats/' },
+ { from: '/streaming-lakehouse/integrate-data-lakes/catalogs/',
to: '/streaming-lakehouse/datalake-catalogs/' },
+ ];
+ for (const rule of renameRules) {
+ if (existingPath.includes(rule.to)) {
+ redirects.push(existingPath.replace(rule.to, rule.from));
+ }
+ }
+
+ return redirects;
},
},
],
