This is an automated email from the ASF dual-hosted git repository.
KevinyhZou pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 526819a79b [GLUTEN-13256][CH] Support reading Paimon tables with
paimon-cpp (#13334)
526819a79b is described below
commit 526819a79b3167129a6e2f8db7e7cda00da82130
Author: kevinyhzou <[email protected]>
AuthorDate: Thu Nov 5 11:08:33 2026 +0800
[GLUTEN-13256][CH] Support reading Paimon tables with paimon-cpp (#13334)
---
.github/workflows/clickhouse_be_trigger.yml | 1 +
backends-clickhouse/pom.xml | 26 ++
.../org.apache.gluten.component.CHPaimonComponent | 1 +
.../gluten/component/CHOffloadPaimonScan.scala | 42 +++
.../gluten/component/CHPaimonComponent.scala | 50 ++++
.../execution/CHPaimonCountTransformer.scala | 62 +++++
.../gluten/execution/CHPaimonScanTransformer.scala | 181 +++++++++++++
.../gluten/execution/ClickHousePaimonSuite.scala | 259 ++++++++++++++++++
cpp-ch/CMakeLists.txt | 72 ++++-
cpp-ch/cmake/BuildPaimon.cmake | 243 +++++++++++++++++
cpp-ch/cmake/CheckPaimonLibcxx.cmake | 34 +++
cpp-ch/cmake/PaimonGlibcCompat.c | 86 ++++++
cpp-ch/cmake/PatchPaimonArrowThrift.cmake | 284 ++++++++++++++++++++
cpp-ch/local-engine/CMakeLists.txt | 13 +
.../Storages/SubstraitSource/CMakeLists.txt | 20 ++
.../Storages/SubstraitSource/FileReader.cpp | 4 +-
.../Storages/SubstraitSource/FormatFile.cpp | 13 +
.../Paimon/HdfsPaimonFileSystem.cpp | 290 +++++++++++++++++++++
.../SubstraitSource/Paimon/HdfsPaimonFileSystem.h | 31 +--
.../SubstraitSource/Paimon/PaimonFormatFile.cpp | 48 ++++
.../SubstraitSource/Paimon/PaimonFormatFile.h | 44 ++--
.../SubstraitSource/Paimon/PaimonReader.cpp | 282 ++++++++++++++++++++
.../Storages/SubstraitSource/Paimon/PaimonReader.h | 29 +--
docs/backends/clickhouse/get-started/ClickHouse.md | 57 +++-
ep/build-clickhouse/src/build-clickhouse.sh | 6 +-
ep/build-clickhouse/src/install-ubuntu.sh | 2 +-
ep/build-clickhouse/src/package.sh | 34 ++-
.../substrait/rel/PaimonLocalFilesBuilder.java | 11 +
.../gluten/substrait/rel/PaimonLocalFilesNode.java | 52 ++++
.../substrait/proto/substrait/algebra.proto | 8 +
gluten-ut/spark35/pom.xml | 11 +
.../substrait/rel/PaimonLocalFilesSuite.scala | 44 ++--
.../gluten/component/CHPaimonComponentSuite.scala | 32 +--
33 files changed, 2230 insertions(+), 142 deletions(-)
diff --git a/.github/workflows/clickhouse_be_trigger.yml
b/.github/workflows/clickhouse_be_trigger.yml
index e09be79627..41ccdb5342 100644
--- a/.github/workflows/clickhouse_be_trigger.yml
+++ b/.github/workflows/clickhouse_be_trigger.yml
@@ -23,6 +23,7 @@ on:
- 'backends-clickhouse/**'
- 'gluten-celeborn/**'
- 'gluten-iceberg/**'
+ - 'gluten-paimon/**'
- 'gluten-core/**'
- 'gluten-substrait/**'
- 'gluten-ut/**'
diff --git a/backends-clickhouse/pom.xml b/backends-clickhouse/pom.xml
index 686ba313ff..6e76e61a91 100644
--- a/backends-clickhouse/pom.xml
+++ b/backends-clickhouse/pom.xml
@@ -497,6 +497,32 @@
</build>
<profiles>
+ <profile>
+ <id>paimon</id>
+ <activation>
+ <activeByDefault>false</activeByDefault>
+ </activation>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.gluten</groupId>
+ <artifactId>gluten-paimon</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.gluten</groupId>
+ <artifactId>gluten-paimon</artifactId>
+ <version>${project.version}</version>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.paimon</groupId>
+
<artifactId>paimon-spark-${sparkbundle.version}_${scala.binary.version}</artifactId>
+ <version>${paimon.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ </dependencies>
+ </profile>
<profile>
<id>celeborn</id>
<activation>
diff --git
a/backends-clickhouse/src-paimon/main/resources/META-INF/gluten-components/org.apache.gluten.component.CHPaimonComponent
b/backends-clickhouse/src-paimon/main/resources/META-INF/gluten-components/org.apache.gluten.component.CHPaimonComponent
new file mode 100644
index 0000000000..8b13789179
--- /dev/null
+++
b/backends-clickhouse/src-paimon/main/resources/META-INF/gluten-components/org.apache.gluten.component.CHPaimonComponent
@@ -0,0 +1 @@
+
diff --git
a/backends-clickhouse/src-paimon/main/scala/org/apache/gluten/component/CHOffloadPaimonScan.scala
b/backends-clickhouse/src-paimon/main/scala/org/apache/gluten/component/CHOffloadPaimonScan.scala
new file mode 100644
index 0000000000..d7bbd05888
--- /dev/null
+++
b/backends-clickhouse/src-paimon/main/scala/org/apache/gluten/component/CHOffloadPaimonScan.scala
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gluten.component
+
+import org.apache.gluten.execution.{CHPaimonCountTransformer,
CHPaimonScanTransformer, PaimonScanTransformer}
+import org.apache.gluten.extension.columnar.offload.OffloadSingleNode
+
+import org.apache.spark.sql.execution.{LocalTableScanExec, SparkPlan}
+import org.apache.spark.sql.execution.datasources.v2.BatchScanExec
+import org.apache.spark.sql.execution.datasources.v2.DataSourceV2ScanRelation
+
+import org.apache.paimon.spark.read.PaimonLocalScan
+
+case object CHOffloadPaimonScan extends OffloadSingleNode {
+ override def offload(plan: SparkPlan): SparkPlan = {
+ plan match {
+ case scan: LocalTableScanExec
+ if scan.logicalLink.exists(_.exists {
+ case relation: DataSourceV2ScanRelation =>
+ relation.scan.isInstanceOf[PaimonLocalScan]
+ case _ => false
+ }) => CHPaimonCountTransformer.replace(scan)
+ case scan: BatchScanExec if
PaimonScanTransformer.supportsBatchScan(scan.scan) =>
+ CHPaimonScanTransformer(scan)
+ case other => other
+ }
+ }
+}
diff --git
a/backends-clickhouse/src-paimon/main/scala/org/apache/gluten/component/CHPaimonComponent.scala
b/backends-clickhouse/src-paimon/main/scala/org/apache/gluten/component/CHPaimonComponent.scala
new file mode 100644
index 0000000000..d6e2cec775
--- /dev/null
+++
b/backends-clickhouse/src-paimon/main/scala/org/apache/gluten/component/CHPaimonComponent.scala
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gluten.component
+
+import org.apache.gluten.backendsapi.clickhouse.CHBackend
+import org.apache.gluten.config.GlutenConfig
+import org.apache.gluten.extension.columnar.heuristic.HeuristicTransform
+import org.apache.gluten.extension.columnar.validator.Validators
+import org.apache.gluten.extension.injector.Injector
+
+import org.apache.spark.util.SparkReflectionUtil
+
+/**
+ * Registered through the resource file named after this class. Gluten
discovers
+ * components by that file name when the Paimon profile is packaged; the marker
+ * file itself is intentionally empty.
+ */
+class CHPaimonComponent extends Component {
+ override def name(): String = "clickhouse-paimon"
+
+ override def dependencies(): Seq[Class[_ <: Component]] = classOf[CHBackend]
:: Nil
+
+ override def isRuntimeCompatible: Boolean = {
+ SparkReflectionUtil.isClassPresent(
+ "org.apache.paimon.spark.extensions.PaimonSparkSessionExtensions")
+ }
+
+ override def injectRules(injector: Injector): Unit = {
+ injector.gluten.legacy.injectTransform { c =>
+ val offload = Seq(CHOffloadPaimonScan)
+ HeuristicTransform.Simple(
+ Validators.newValidator(new GlutenConfig(c.sqlConf), offload),
+ offload)
+ }
+ }
+}
diff --git
a/backends-clickhouse/src-paimon/main/scala/org/apache/gluten/execution/CHPaimonCountTransformer.scala
b/backends-clickhouse/src-paimon/main/scala/org/apache/gluten/execution/CHPaimonCountTransformer.scala
new file mode 100644
index 0000000000..d431ababb0
--- /dev/null
+++
b/backends-clickhouse/src-paimon/main/scala/org/apache/gluten/execution/CHPaimonCountTransformer.scala
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gluten.execution
+
+import org.apache.spark.rdd.RDD
+import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.catalyst.expressions.{Attribute, SortOrder}
+import org.apache.spark.sql.catalyst.plans.physical.Partitioning
+import org.apache.spark.sql.execution.{LocalTableScanExec,
LocalTableScanTransformer, RowToCHNativeColumnarExec, SparkPlan}
+import org.apache.spark.sql.vectorized.ColumnarBatch
+
+/** Converts an already computed Paimon count result into a CH native batch. */
+case class CHPaimonCountTransformer(
+ outputAttributes: Seq[Attribute],
+ @transient rows: Seq[InternalRow],
+ override val outputPartitioning: Partitioning,
+ override val outputOrdering: Seq[SortOrder])
+ extends LocalTableScanTransformer(outputAttributes, outputPartitioning,
outputOrdering) {
+
+ override protected def doValidateInternal(): ValidationResult = {
+ if (rows == null) {
+ ValidationResult.failed("Deserialized Paimon count scan has no rows")
+ } else {
+ ValidationResult.succeeded
+ }
+ }
+
+ override def doExecuteColumnar(): RDD[ColumnarBatch] = {
+ if (rows == null) {
+ throw new IllegalStateException("Deserialized Paimon count scan has no
rows")
+ }
+ RowToCHNativeColumnarExec(LocalTableScanExec(output,
rows)).executeColumnar()
+ }
+
+ override protected def withNewChildrenInternal(newChildren:
IndexedSeq[SparkPlan]): SparkPlan = {
+ assert(newChildren.isEmpty)
+ copy(outputAttributes, rows, outputPartitioning, outputOrdering)
+ }
+}
+
+object CHPaimonCountTransformer {
+ def replace(plan: LocalTableScanExec): LocalTableScanTransformer =
+ CHPaimonCountTransformer(
+ plan.output,
+ plan.rows,
+ plan.outputPartitioning,
+ plan.outputOrdering)
+}
diff --git
a/backends-clickhouse/src-paimon/main/scala/org/apache/gluten/execution/CHPaimonScanTransformer.scala
b/backends-clickhouse/src-paimon/main/scala/org/apache/gluten/execution/CHPaimonScanTransformer.scala
new file mode 100644
index 0000000000..baadfc5c43
--- /dev/null
+++
b/backends-clickhouse/src-paimon/main/scala/org/apache/gluten/execution/CHPaimonScanTransformer.scala
@@ -0,0 +1,181 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gluten.execution
+
+import org.apache.gluten.exception.GlutenNotSupportException
+import org.apache.gluten.sql.shims.SparkShimLoader
+import org.apache.gluten.substrait.rel.{PaimonLocalFilesBuilder, SplitInfo}
+import org.apache.gluten.substrait.rel.LocalFilesNode.ReadFileFormat
+
+import org.apache.spark.Partition
+import org.apache.spark.rdd.RDD
+import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.catalyst.expressions.{AttributeReference,
DynamicPruningExpression, Expression, Literal}
+import org.apache.spark.sql.catalyst.plans.QueryPlan
+import org.apache.spark.sql.connector.catalog.Table
+import org.apache.spark.sql.connector.read.Scan
+import org.apache.spark.sql.execution.datasources.v2.BatchScanExec
+import org.apache.spark.sql.types.StructType
+import org.apache.spark.sql.vectorized.ColumnarBatch
+
+import org.apache.paimon.CoreOptions
+import org.apache.paimon.io.DataOutputSerializer
+import org.apache.paimon.spark.{PaimonInputPartition, PaimonScan}
+import org.apache.paimon.table.FileStoreTable
+import org.apache.paimon.table.source.DataSplit
+
+import scala.collection.JavaConverters._
+
+case class CHPaimonScanTransformer(
+ override val output: Seq[AttributeReference],
+ @transient override val scan: Scan,
+ override val runtimeFilters: Seq[Expression],
+ @transient override val table: Table,
+ override val keyGroupedPartitioning: Option[Seq[Expression]] = None,
+ override val commonPartitionValues: Option[Seq[(InternalRow, Int)]] = None,
+ override val pushDownFilters: Option[Seq[Expression]] = None)
+ extends BatchScanExecTransformerBase(
+ output = output,
+ scan = scan,
+ runtimeFilters = runtimeFilters,
+ table = table,
+ keyGroupedPartitioning = keyGroupedPartitioning,
+ commonPartitionValues = commonPartitionValues
+ ) {
+
+ private lazy val paimonScanTable: FileStoreTable = scan match {
+ case paimonScan: PaimonScan =>
+ paimonScan.table match {
+ case dataTable: FileStoreTable => dataTable
+ case _ => throw new GlutenNotSupportException("Only support Paimon
DataTable.")
+ }
+ case _ => throw new GlutenNotSupportException("Only support PaimonScan.")
+ }
+
+ private lazy val coreOptions: CoreOptions = paimonScanTable.coreOptions()
+
+ // Paimon folds query options into the scan table, including its schema
options.
+ private[execution] lazy val effectiveReadOptions: Map[String, String] =
+ paimonScanTable.options().asScala.toMap
+
+ override def getPartitionSchema: StructType = scan match {
+ case paimonScan: PaimonScan =>
+ val partitionKeys = paimonScan.table.partitionKeys()
+ StructType(scan.readSchema().filter(field =>
partitionKeys.contains(field.name)))
+ case _ => throw new GlutenNotSupportException("Only support PaimonScan.")
+ }
+
+ override def getDataSchema: StructType = scan match {
+ case paimonScan: PaimonScan =>
+ val partitionKeys = paimonScan.table.partitionKeys()
+ StructType(scan.readSchema().filterNot(field =>
partitionKeys.contains(field.name)))
+ case _ => throw new GlutenNotSupportException("Only support PaimonScan.")
+ }
+
+ override def withNewPushdownFilters(filters: Seq[Expression]):
CHPaimonScanTransformer =
+ copy(pushDownFilters = Some(filters))
+
+ override def withOutput(newOutput: Seq[AttributeReference]):
BatchScanExecTransformerBase =
+ copy(output = newOutput)
+
+ override lazy val fileFormat: ReadFileFormat = {
+ val formatStr = coreOptions.fileFormatString()
+ if ("parquet".equalsIgnoreCase(formatStr)) {
+ ReadFileFormat.ParquetReadFormat
+ } else if ("orc".equalsIgnoreCase(formatStr)) {
+ ReadFileFormat.OrcReadFormat
+ } else {
+ ReadFileFormat.UnknownFormat
+ }
+ }
+
+ override def doValidateInternal(): ValidationResult = {
+ scan match {
+ case paimonScan: PaimonScan =>
+ paimonScan.table match {
+ case _: FileStoreTable =>
+ if (!paimonScan.inputSplits.forall(_.isInstanceOf[DataSplit])) {
+ return ValidationResult.failed("Native Paimon reader only
supports DataSplit.")
+ }
+ case table =>
+ return ValidationResult.failed(
+ s"Not support Paimon ${table.getClass.getSimpleName} table.")
+ }
+ case _ => return ValidationResult.failed("Only support PaimonScan.")
+ }
+ super.doValidateInternal()
+ }
+
+ override def doExecuteColumnar(): RDD[ColumnarBatch] = throw new
UnsupportedOperationException()
+
+ override def getSplitInfosFromPartitions(
+ partitions: Seq[(Partition, ReadFileFormat)]): Seq[SplitInfo] = {
+ partitions.map { case (partition, _) =>
+ partition match {
+ case p: SparkDataSourceRDDPartition =>
+ val splits = p.inputPartitions.flatMap {
+ case input: PaimonInputPartition =>
+ input.splits.map {
+ case split: DataSplit =>
+ val out = new DataOutputSerializer(1024)
+ split.serialize(out)
+ out.getCopyOfBuffer
+ case other =>
+ throw new GlutenNotSupportException(s"Unsupported Paimon
split: $other")
+ }
+ case other =>
+ throw new GlutenNotSupportException(s"Unsupported input
partition type: $other")
+ }
+ val node = PaimonLocalFilesBuilder.makeNativePaimonLocalFiles(
+ p.index,
+ coreOptions.path().toString,
+ coreOptions.branch(),
+ effectiveReadOptions.asJava,
+ splits.asJava,
+ p.preferredLocations().toList.asJava)
+ node.setFileSchema(getDataSchema)
+ node
+ case _ => throw new GlutenNotSupportException()
+ }
+ }
+ }
+
+ override def doCanonicalize(): CHPaimonScanTransformer = {
+ copy(
+ output = output.map(QueryPlan.normalizeExpressions(_, output)),
+ runtimeFilters = QueryPlan.normalizePredicates(
+ runtimeFilters.filterNot(_ ==
DynamicPruningExpression(Literal.TrueLiteral)),
+ output),
+ pushDownFilters = pushDownFilters.map(QueryPlan.normalizePredicates(_,
output))
+ )
+ }
+
+ override protected[this] def supportsBatchScan(scan: Scan): Boolean =
+ PaimonScanTransformer.supportsBatchScan(scan)
+}
+
+object CHPaimonScanTransformer {
+ def apply(batchScan: BatchScanExec): CHPaimonScanTransformer = {
+ new CHPaimonScanTransformer(
+ batchScan.output,
+ batchScan.scan,
+ batchScan.runtimeFilters,
+ table = batchScan.table,
+ keyGroupedPartitioning = batchScan.keyGroupedPartitioning,
+ commonPartitionValues =
SparkShimLoader.getSparkShims.getCommonPartitionValues(batchScan))
+ }
+}
diff --git
a/backends-clickhouse/src-paimon/test/scala/org/apache/gluten/execution/ClickHousePaimonSuite.scala
b/backends-clickhouse/src-paimon/test/scala/org/apache/gluten/execution/ClickHousePaimonSuite.scala
new file mode 100644
index 0000000000..ad84fcd019
--- /dev/null
+++
b/backends-clickhouse/src-paimon/test/scala/org/apache/gluten/execution/ClickHousePaimonSuite.scala
@@ -0,0 +1,259 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gluten.execution
+
+import org.apache.gluten.config.GlutenConfig
+
+import org.apache.spark.SparkConf
+import org.apache.spark.sql.{DataFrame, Row}
+
+class ClickHousePaimonSuite extends GlutenClickHouseWholeStageTransformerSuite
{
+ override protected def sparkConf: SparkConf = {
+ super.sparkConf
+ .set("spark.sql.adaptive.enabled", "false")
+ .set("spark.sql.extensions",
"org.apache.paimon.spark.extensions.PaimonSparkSessionExtensions")
+ .set("spark.sql.catalog.spark_catalog",
"org.apache.paimon.spark.SparkCatalog")
+ .set("spark.sql.catalog.spark_catalog.warehouse",
s"file://$dataHome/data-paimon")
+ }
+
+ private def checkNativePaimonScan(df: DataFrame): Unit = {
+ checkGlutenPlan[BatchScanExecTransformerBase](df)
+ assert(df.queryExecution.executedPlan.exists {
+ case stage: WholeStageTransformer =>
+ stage.child.exists {
+ case _: PaimonScanTransformer | _: CHPaimonScanTransformer => true
+ case _ => false
+ }
+ case _ => false
+ })
+ }
+
+ Seq("parquet", "orc").foreach { format =>
+ test(s"read Paimon $format table with ClickHouse") {
+ assume(sys.env.get("GLUTEN_TEST_NATIVE_PAIMON").contains("1"))
+ withTable("paimon_ch_table") {
+ withSQLConf(GlutenConfig.GLUTEN_ENABLED.key -> "false") {
+ sql(s"CREATE TABLE paimon_ch_table (id INT, name STRING) USING
paimon " +
+ s"TBLPROPERTIES ('file.format'='$format')")
+ sql("INSERT INTO paimon_ch_table VALUES (1, 'Bob'), (2, 'Alice')")
+ }
+ runQueryAndCompare("SELECT id, name FROM paimon_ch_table ORDER BY id")
{
+ checkNativePaimonScan
+ }
+ }
+ }
+ }
+
+ test("read non-primary-key Paimon table using native SDK") {
+ assume(sys.env.get("GLUTEN_TEST_NATIVE_PAIMON").contains("1"))
+ withTable("paimon_ch_native_read") {
+ withSQLConf(GlutenConfig.GLUTEN_ENABLED.key -> "false") {
+ sql("CREATE TABLE paimon_ch_native_read (id INT, name STRING) USING
paimon " +
+ "TBLPROPERTIES ('bucket'='-1', 'file.format'='parquet')")
+ sql("INSERT INTO paimon_ch_native_read VALUES (1, 'Bob'), (2,
'Alice')")
+ }
+
+ runQueryAndCompare("SELECT id, name FROM paimon_ch_native_read ORDER BY
id") { df =>
+ checkNativePaimonScan(df)
+ assert(df.queryExecution.executedPlan.exists {
+ case _: CHPaimonScanTransformer => true
+ case _ => false
+ })
+ }
+ }
+ }
+
+ test("native Paimon scan preserves branch") {
+ assume(sys.env.get("GLUTEN_TEST_NATIVE_PAIMON").contains("1"))
+ withTable("paimon_ch_branch") {
+ withSQLConf(GlutenConfig.GLUTEN_ENABLED.key -> "false") {
+ sql("CREATE TABLE paimon_ch_branch (id INT) USING paimon " +
+ "TBLPROPERTIES ('bucket'='-1', 'file.format'='parquet')")
+ sql("INSERT INTO paimon_ch_branch VALUES (1)")
+ sql("CALL sys.create_tag('default.paimon_ch_branch', 'review_tag')")
+ sql("CALL sys.create_branch('default.paimon_ch_branch',
'review_branch', 'review_tag')")
+ sql("INSERT INTO paimon_ch_branch VALUES (2)")
+ }
+
+ val path = s"file://$dataHome/data-paimon/default.db/paimon_ch_branch"
+ def readBranch(): DataFrame = spark.read.format("paimon")
+ .option("path", path)
+ .option("branch", "review_branch")
+ .load()
+ withSQLConf(GlutenConfig.GLUTEN_ENABLED.key -> "false") {
+ assert(readBranch().collect().toSeq == Seq(Row(1)))
+ }
+ val df = readBranch()
+ assert(df.collect().toSeq == Seq(Row(1)))
+ val nativeScans = df.queryExecution.executedPlan.collect {
+ case scan: CHPaimonScanTransformer => scan
+ }
+ assert(nativeScans.nonEmpty, df.queryExecution.executedPlan.toString())
+ }
+ }
+
+ test("native Paimon scan applies a query-level read-option override") {
+ assume(sys.env.get("GLUTEN_TEST_NATIVE_PAIMON").contains("1"))
+ withTable("paimon_ch_read_options") {
+ withSQLConf(GlutenConfig.GLUTEN_ENABLED.key -> "false") {
+ sql("CREATE TABLE paimon_ch_read_options (id INT) USING paimon " +
+ "TBLPROPERTIES ('bucket'='-1', 'file.format'='parquet',
'read.batch-size'='64')")
+ sql("INSERT INTO paimon_ch_read_options VALUES (1), (2)")
+ }
+
+ val path =
s"file://$dataHome/data-paimon/default.db/paimon_ch_read_options"
+ val df = spark.read.format("paimon")
+ .option("path", path)
+ .option("read.batch-size", "1")
+ .load()
+ assert(df.collect().toSet == Set(Row(1), Row(2)))
+ val nativeScans = df.queryExecution.executedPlan.collect {
+ case scan: CHPaimonScanTransformer => scan
+ }
+ assert(nativeScans.nonEmpty, df.queryExecution.executedPlan.toString())
+ nativeScans.foreach { scan =>
+ assert(scan.effectiveReadOptions.get("read.batch-size").contains("1"))
+ }
+ }
+ }
+
+ test("incremental diff scan falls back to Paimon JVM reader") {
+ withTable("paimon_ch_incremental_fallback") {
+ withSQLConf(GlutenConfig.GLUTEN_ENABLED.key -> "false") {
+ sql("CREATE TABLE paimon_ch_incremental_fallback " +
+ "(id INT, name STRING) USING paimon " +
+ "TBLPROPERTIES ('primary-key'='id', 'bucket'='1',
'file.format'='parquet')")
+ sql("INSERT INTO paimon_ch_incremental_fallback VALUES (1, 'old')")
+ sql("INSERT INTO paimon_ch_incremental_fallback VALUES (1, 'new')")
+ }
+
+ val path =
s"file://$dataHome/data-paimon/default.db/paimon_ch_incremental_fallback"
+ val df = spark.read.format("paimon")
+ .option("path", path)
+ .option("incremental-between", "1,2")
+ .option("incremental-between-scan-mode", "diff")
+ .load()
+ assert(df.collect().toSeq == Seq(Row(1, "new")))
+ assert(!df.queryExecution.executedPlan.exists {
+ case _: CHPaimonScanTransformer => true
+ case _ => false
+ })
+ }
+ }
+
+ test("sum one over append-only Paimon rows in ClickHouse") {
+ assume(sys.env.get("GLUTEN_TEST_NATIVE_PAIMON").contains("1"))
+ withTable("paimon_ch_native_sum_one") {
+ withSQLConf(GlutenConfig.GLUTEN_ENABLED.key -> "false") {
+ sql("CREATE TABLE paimon_ch_native_sum_one (id STRING, p STRING) USING
paimon " +
+ "PARTITIONED BY (p) TBLPROPERTIES ('bucket'='-1',
'file.format'='parquet')")
+ sql("INSERT INTO paimon_ch_native_sum_one VALUES ('1', 'view'), " +
+ "(NULL, 'view'), ('3', 'click')")
+ }
+
+ runQueryAndCompare(
+ "SELECT sum(1) FROM paimon_ch_native_sum_one WHERE p = 'view'") { df =>
+ assert(df.collect().toSeq == Seq(Row(2L)))
+ assert(df.queryExecution.executedPlan.exists {
+ case _: CHPaimonScanTransformer => true
+ case _ => false
+ })
+ }
+ assert(sql("SELECT sum(1) FROM paimon_ch_native_sum_one WHERE p =
'missing'")
+ .collect().toSeq == Seq(Row(null)))
+ }
+ }
+
+ test("convert Paimon local count result to a CH batch") {
+ withTable("paimon_ch_local_count") {
+ withSQLConf(GlutenConfig.GLUTEN_ENABLED.key -> "false") {
+ sql("CREATE TABLE paimon_ch_local_count (id STRING, p STRING) USING
paimon " +
+ "PARTITIONED BY (p) TBLPROPERTIES ('bucket'='-1',
'file.format'='parquet')")
+ sql("INSERT INTO paimon_ch_local_count VALUES ('1', 'view'), ('2',
'view')")
+ }
+
+ runQueryAndCompare(
+ "SELECT count(1) FROM paimon_ch_local_count WHERE p = 'view'") { df =>
+ assert(df.collect().toSeq == Seq(Row(2L)))
+ assert(df.queryExecution.executedPlan.exists {
+ case _: CHPaimonCountTransformer => true
+ case _ => false
+ })
+ }
+ }
+ }
+
+ test("read partition columns from Paimon table with ClickHouse") {
+ assume(sys.env.get("GLUTEN_TEST_NATIVE_PAIMON").contains("1"))
+ withTable("paimon_ch_partitioned") {
+ withSQLConf(GlutenConfig.GLUTEN_ENABLED.key -> "false") {
+ sql("CREATE TABLE paimon_ch_partitioned (id INT, p STRING) " +
+ "USING paimon PARTITIONED BY (p)")
+ sql("INSERT INTO paimon_ch_partitioned VALUES (1, 'a'), (2, 'b')")
+ }
+ runQueryAndCompare("SELECT id, p FROM paimon_ch_partitioned ORDER BY
id") {
+ checkNativePaimonScan
+ }
+ }
+ }
+
+ test("offload primary key table by default") {
+ assume(sys.env.get("GLUTEN_TEST_NATIVE_PAIMON").contains("1"))
+ withTable("paimon_ch_native_pk") {
+ withSQLConf(GlutenConfig.GLUTEN_ENABLED.key -> "false") {
+ sql("CREATE TABLE paimon_ch_native_pk (id INT, name STRING) USING
paimon " +
+ "TBLPROPERTIES ('primary-key'='id', 'bucket'='1', " +
+ "'file.format'='parquet', 'write-only'='true')")
+ sql("INSERT INTO paimon_ch_native_pk VALUES (1, 'old')")
+ sql("INSERT INTO paimon_ch_native_pk VALUES (1, 'new')")
+ }
+
+ runQueryAndCompare("SELECT id, name FROM paimon_ch_native_pk") { df =>
+ assert(df.collect().toSeq == Seq(Row(1, "new")))
+ checkNativePaimonScan(df)
+ assert(df.queryExecution.executedPlan.exists {
+ case _: CHPaimonScanTransformer => true
+ case _ => false
+ })
+ }
+ }
+ }
+
+ test("count non-null primary key rows with native Paimon reader") {
+ assume(sys.env.get("GLUTEN_TEST_NATIVE_PAIMON").contains("1"))
+ withTable("paimon_ch_native_pk_count") {
+ withSQLConf(GlutenConfig.GLUTEN_ENABLED.key -> "false") {
+ sql("CREATE TABLE paimon_ch_native_pk_count (id INT NOT NULL, p STRING
NOT NULL, " +
+ "name STRING) USING paimon PARTITIONED BY (p) " +
+ "TBLPROPERTIES ('primary-key'='p,id', 'bucket'='1', " +
+ "'file.format'='parquet', 'write-only'='true')")
+ sql("INSERT INTO paimon_ch_native_pk_count VALUES (1, 'view', 'old')")
+ sql("INSERT INTO paimon_ch_native_pk_count VALUES (1, 'view', 'new')")
+ sql("INSERT INTO paimon_ch_native_pk_count VALUES (2, 'click',
'other')")
+ }
+
+ Seq("COUNT(id)", "COUNT(*)").foreach { count =>
+ runQueryAndCompare(
+ s"SELECT $count FROM paimon_ch_native_pk_count WHERE p = 'view'") {
df =>
+ assert(df.collect().toSeq == Seq(Row(1L)))
+ checkNativePaimonScan(df)
+ }
+ }
+ }
+ }
+
+}
diff --git a/cpp-ch/CMakeLists.txt b/cpp-ch/CMakeLists.txt
index 596c3649e9..42b9a5c67c 100644
--- a/cpp-ch/CMakeLists.txt
+++ b/cpp-ch/CMakeLists.txt
@@ -84,6 +84,54 @@ execute_process(
set(CH_BINARY_DIR "${CMAKE_CURRENT_SOURCE_DIR}/build")
option(ENABLE_CPP_TEST "Build CPP Unit test" OFF)
+option(GLUTEN_ENABLE_NATIVE_PAIMON
+ "Build libch with native Paimon read support" ON)
+option(
+ GLUTEN_BUNDLE_PAIMON
+ "Download and build paimon-cpp into build/deps/paimon when Paimon_DIR is
unset"
+ ON)
+
+set(GLUTEN_USE_BUNDLED_PAIMON OFF)
+set(_GLUTEN_PAIMON_BUILD_DEP "")
+set(_GLUTEN_CH_CMAKE_PAIMON_ARG
+ "-DGLUTEN_ENABLE_NATIVE_PAIMON=${GLUTEN_ENABLE_NATIVE_PAIMON}")
+
+if(GLUTEN_ENABLE_NATIVE_PAIMON)
+ if(Paimon_DIR)
+ set(GLUTEN_PAIMON_CMAKE_DIR "${Paimon_DIR}")
+ message(STATUS "Using external paimon-cpp from Paimon_DIR=${Paimon_DIR}")
+ elseif(GLUTEN_BUNDLE_PAIMON)
+ include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/BuildPaimon.cmake)
+ set(GLUTEN_USE_BUNDLED_PAIMON ON)
+ message(
+ STATUS "Bundled paimon-cpp will be installed to
${PAIMON_INSTALL_PREFIX}")
+ else()
+ message(
+ FATAL_ERROR
+ "Native Paimon requires paimon-cpp. Set -DPaimon_DIR=... or
-DGLUTEN_BUNDLE_PAIMON=ON."
+ )
+ endif()
+ if(GLUTEN_USE_BUNDLED_PAIMON)
+ # The bridge sources are compiled into libch. Keep Paimon itself in shared
+ # libraries so its implementation is not copied into libch.
+ set(_GLUTEN_PAIMON_BUILD_DEP paimon_cpp)
+ list(APPEND _GLUTEN_CH_CMAKE_PAIMON_ARG
+ "-DPaimon_DIR=${GLUTEN_PAIMON_CMAKE_DIR}"
+ "-DPAIMON_BUNDLED_DEPS_DIR=${PAIMON_EP_BINARY_DIR}")
+ else()
+ message(
+ FATAL_ERROR
+ "Native Paimon support requires bundled paimon-cpp
(-DGLUTEN_BUNDLE_PAIMON=ON)."
+ )
+ endif()
+else()
+ message(
+ STATUS
+ "GLUTEN_ENABLE_NATIVE_PAIMON=OFF: libch builds without paimon-cpp (raw
Parquet Paimon scans only)."
+ )
+endif()
+
+set(LIBCH_OUTPUT_DIR "${CH_BINARY_DIR}/utils/extern-local-engine")
if(ENABLE_CPP_TEST)
add_custom_command(
@@ -93,11 +141,12 @@ if(ENABLE_CPP_TEST)
-DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} -DNO_ARMV81_OR_HIGHER=1
-DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DENABLE_PROTOBUF=ON
-DENABLE_TESTS=ON -DENABLE_JEMALLOC=ON -DENABLE_MULTITARGET_CODE=ON
- -DENABLE_EXTERN_LOCAL_ENGINE=ON -DCOMPILER_FLAGS='-fvisibility=hidden
- -fvisibility-inlines-hidden' -S ${CH_SOURCE_DIR} -G Ninja -B
- ${CH_BINARY_DIR} && cmake --build ${CH_BINARY_DIR} --target ch
- unit_tests_local_engine\"
- OUTPUT _build_ch)
+ -DENABLE_EXTERN_LOCAL_ENGINE=ON ${_GLUTEN_CH_CMAKE_PAIMON_ARG}
+ -DCOMPILER_FLAGS='-fvisibility=hidden -fvisibility-inlines-hidden' -S
+ ${CH_SOURCE_DIR} -G Ninja -B ${CH_BINARY_DIR} && cmake --build
+ ${CH_BINARY_DIR} --target ch unit_tests_local_engine\"
+ OUTPUT _build_ch
+ DEPENDS ${_GLUTEN_PAIMON_BUILD_DEP})
else()
add_custom_command(
USES_TERMINAL
@@ -114,11 +163,18 @@ else()
-DENABLE_MYSQL=OFF -DENABLE_BCRYPT=OFF -DENABLE_LDAP=OFF
-DENABLE_MSGPACK=OFF -DUSE_REPLXX=OFF -DENABLE_CLICKHOUSE_ALL=OFF
-DENABLE_GWP_ASAN=OFF -DENABLE_NUMACTL=OFF -DENABLE_GOOGLE_CLOUD_CPP=OFF
- -DENABLE_ARROW_FLIGHT=OFF -DENABLE_XRAY=0
+ -DENABLE_ARROW_FLIGHT=OFF -DENABLE_XRAY=0 ${_GLUTEN_CH_CMAKE_PAIMON_ARG}
-DCOMPILER_FLAGS='-fvisibility=hidden -fvisibility-inlines-hidden' -S
${CH_SOURCE_DIR} -G Ninja -B ${CH_BINARY_DIR} && cmake --build
${CH_BINARY_DIR} --target libch\"
- OUTPUT _build_ch)
+ OUTPUT _build_ch
+ DEPENDS ${_GLUTEN_PAIMON_BUILD_DEP})
endif()
-add_custom_target(build_ch ALL DEPENDS _build_ch)
+if(GLUTEN_USE_BUNDLED_PAIMON)
+ gluten_bundle_paimon_libs("${LIBCH_OUTPUT_DIR}"
+ "${CMAKE_CURRENT_SOURCE_DIR}/..")
+ add_custom_target(build_ch ALL DEPENDS bundle_paimon_libs)
+else()
+ add_custom_target(build_ch ALL DEPENDS _build_ch)
+endif()
diff --git a/cpp-ch/cmake/BuildPaimon.cmake b/cpp-ch/cmake/BuildPaimon.cmake
new file mode 100644
index 0000000000..d38ac8c28a
--- /dev/null
+++ b/cpp-ch/cmake/BuildPaimon.cmake
@@ -0,0 +1,243 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Download, build, and install paimon-cpp before libch's native Paimon reader.
+# Use ClickHouse's libc++ runtime while keeping Paimon's own Arrow dependency.
+include(ExternalProject)
+
+# Paimon's bundled Arrow patch is applied with GNU patch. Some CI builder
images
+# do not include it, while the build container has apt available.
+find_program(GLUTEN_PATCH_EXECUTABLE patch)
+if(NOT GLUTEN_PATCH_EXECUTABLE)
+ execute_process(COMMAND apt-get update
+ RESULT_VARIABLE _patch_apt_update_result)
+ if(NOT _patch_apt_update_result EQUAL 0)
+ message(
+ FATAL_ERROR
+ "GNU patch is required to build Paimon, and apt-get update failed")
+ endif()
+ execute_process(COMMAND apt-get install -y patch
+ RESULT_VARIABLE _patch_apt_install_result)
+ if(NOT _patch_apt_install_result EQUAL 0)
+ message(
+ FATAL_ERROR
+ "GNU patch is required to build Paimon, and apt-get install failed")
+ endif()
+ unset(GLUTEN_PATCH_EXECUTABLE CACHE)
+ find_program(GLUTEN_PATCH_EXECUTABLE patch)
+endif()
+if(NOT GLUTEN_PATCH_EXECUTABLE)
+ message(FATAL_ERROR "GNU patch is required to build Paimon")
+endif()
+
+set(PAIMON_INSTALL_PREFIX
+ "${CMAKE_CURRENT_SOURCE_DIR}/build/deps/paimon"
+ CACHE PATH "Install prefix for bundled paimon-cpp")
+set(PAIMON_GIT_REPOSITORY
+ "https://github.com/apache/paimon-cpp.git"
+ CACHE STRING "paimon-cpp git repository")
+set(PAIMON_GIT_TAG
+ "v0.3.0"
+ CACHE STRING "paimon-cpp git tag (must match find_package version 0.3.0)")
+
+string(REGEX REPLACE "[^A-Za-z0-9_.-]" "_" PAIMON_BUILD_TAG
"${PAIMON_GIT_TAG}")
+set(PAIMON_EP_SOURCE_DIR
+
"${CMAKE_CURRENT_SOURCE_DIR}/build/deps/paimon-${PAIMON_BUILD_TAG}-src-upstream-arrow"
+)
+set(PAIMON_EP_BINARY_DIR
+
"${CMAKE_CURRENT_SOURCE_DIR}/build/deps/paimon-${PAIMON_BUILD_TAG}-build-upstream-arrow-sysroot"
+)
+# Read ClickHouse's toolchain in a function so its settings do not change the
+# outer Gluten build. Keep the sysroot in the compiler flags as well: Paimon
and
+# Arrow forward C/CXX flags to their nested ExternalProject dependencies, but
do
+# not forward CMAKE_SYSROOT or CMAKE_TOOLCHAIN_FILE.
+function(gluten_paimon_sysroot_flags)
+ string(TOLOWER "${CMAKE_SYSTEM_PROCESSOR}" _processor)
+ if(_processor MATCHES "^(amd64|x86_64)$")
+ set(_processor x86_64)
+ elseif(_processor MATCHES "^(arm64|aarch64)$")
+ set(_processor aarch64)
+ else()
+ message(
+ FATAL_ERROR "Unsupported Paimon sysroot architecture: ${_processor}")
+ endif()
+ set(_toolchain "${CH_SOURCE_DIR}/cmake/linux/toolchain-${_processor}.cmake")
+ include("${_toolchain}")
+ if(NOT EXISTS "${CMAKE_SYSROOT}/usr/include/stdlib.h")
+ message(
+ FATAL_ERROR "ClickHouse sysroot is not initialized: ${CMAKE_SYSROOT}")
+ endif()
+ set(GLUTEN_PAIMON_SYSROOT
+ "${CMAKE_SYSROOT}"
+ PARENT_SCOPE)
+ set(GLUTEN_PAIMON_BUILTINS_ARCHIVE
+
"${CH_BINARY_DIR}/clang-builtins/lib/linux/libclang_rt.builtins-${_processor}.a"
+ PARENT_SCOPE)
+ # Do not let Clang locate runtime libraries in its system resource directory:
+ # CI may have Clang installed without compiler-rt. ClickHouse builds these
+ # builtins before Paimon is configured. Forward them in the compiler flags so
+ # nested dependency builds (including C-only executables) use them too.
+ set(GLUTEN_PAIMON_SYSROOT_FLAGS
+ "--target=${CMAKE_CXX_COMPILER_TARGET} --sysroot=${CMAKE_SYSROOT}
--gcc-toolchain=${TOOLCHAIN_PATH} -fuse-ld=lld -nodefaultlibs
-Wl,--start-group,${CH_BINARY_DIR}/clang-builtins/lib/linux/libclang_rt.builtins-${_processor}.a,${CH_BINARY_DIR}/contrib/libunwind-cmake/libunwind.a,--end-group
-lc -lm -lpthread -ldl"
+ PARENT_SCOPE)
+endfunction()
+gluten_paimon_sysroot_flags()
+set(GLUTEN_PAIMON_LIBCXX_INCLUDE
+
"${CMAKE_CURRENT_SOURCE_DIR}/build/contrib/llvm-project/libcxx/include/c++/v1"
+)
+set(GLUTEN_PAIMON_LIBCXX_SOURCE_INCLUDE
+
"${CMAKE_CURRENT_SOURCE_DIR}/ClickHouse/contrib/llvm-project/libcxx/include"
+)
+set(GLUTEN_PAIMON_LIBCXXABI_INCLUDE
+
"${CMAKE_CURRENT_SOURCE_DIR}/ClickHouse/contrib/llvm-project/libcxxabi/include"
+)
+set(GLUTEN_PAIMON_LIBCXX_ARCHIVE
+ "${CMAKE_CURRENT_SOURCE_DIR}/build/contrib/libcxx-cmake/libcxx.a")
+set(GLUTEN_PAIMON_LIBCXXABI_ARCHIVE
+ "${CMAKE_CURRENT_SOURCE_DIR}/build/contrib/libcxxabi-cmake/libcxxabi.a")
+set(GLUTEN_PAIMON_LIBUNWIND_ARCHIVE
+ "${CMAKE_CURRENT_SOURCE_DIR}/build/contrib/libunwind-cmake/libunwind.a")
+# Prefer Paimon's bundled Arrow and fmt headers over system installations;
+# ClickHouse supplies only the libc++ runtime and uses its own Arrow build.
+# Match ClickHouse's modified std::exception layout. This flag must also reach
+# every bundled C++ dependency through Paimon's EP_CXX_FLAGS; otherwise runtime
+# constructors write past the smaller exception objects allocated by callers.
+set(GLUTEN_PAIMON_CXX_FLAGS
+ "${GLUTEN_PAIMON_SYSROOT_FLAGS} -DSTD_EXCEPTION_HAS_STACK_TRACE=1
-stdlib=libc++ -nostdlib++ -nostdinc++
-I${PAIMON_EP_BINARY_DIR}/arrow_ep-install/include
-I${PAIMON_EP_BINARY_DIR}/fmt_ep-install/include -isystem
${GLUTEN_PAIMON_LIBCXX_INCLUDE} -isystem ${GLUTEN_PAIMON_LIBCXX_SOURCE_INCLUDE}
-isystem ${GLUTEN_PAIMON_LIBCXXABI_INCLUDE} -include cstdlib"
+)
+set(GLUTEN_PAIMON_CXX_STANDARD_LIBRARIES
+ "${GLUTEN_PAIMON_LIBCXX_ARCHIVE} ${GLUTEN_PAIMON_LIBCXXABI_ARCHIVE}
${GLUTEN_PAIMON_LIBUNWIND_ARCHIVE} -ldl -pthread -lm"
+)
+if(NOT CMAKE_CXX_COMPILER_ID MATCHES "Clang")
+ message(FATAL_ERROR "Native Paimon requires Clang to build with libc++")
+endif()
+# Paimon links ClickHouse's C++ runtime. A clean build does not have these
+# generated headers or archives yet, so prepare them before paimon_cpp starts.
+add_custom_target(
+ prepare_paimon_libcxx
+ COMMAND
+ ${CMAKE_COMMAND} -G Ninja -S ${CH_SOURCE_DIR} -B ${CH_BINARY_DIR}
+ -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}
+ -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
+ -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DNO_ARMV81_OR_HIGHER=1
+ -DENABLE_PROTOBUF=ON -DENABLE_TESTS=OFF -DENABLE_JEMALLOC=ON
+ -DENABLE_MULTITARGET_CODE=ON -DENABLE_EXTERN_LOCAL_ENGINE=ON
+ -DGLUTEN_ENABLE_NATIVE_PAIMON=OFF -DENABLE_CAPNP=OFF -DENABLE_GRPC=OFF
+ -DENABLE_RUST=OFF -DENABLE_H3=OFF -DENABLE_AMQPCPP=OFF
+ -DENABLE_CASSANDRA=OFF -DENABLE_KAFKA=ON -DENABLE_NATS=OFF
+ -DENABLE_LIBPQXX=OFF -DENABLE_NURAFT=OFF -DENABLE_DATASKETCHES=OFF
+ -DENABLE_SQLITE=OFF -DENABLE_S2_GEOMETRY=OFF -DENABLE_ULID=OFF
+ -DENABLE_MYSQL=OFF -DENABLE_BCRYPT=OFF -DENABLE_LDAP=OFF
+ -DENABLE_MSGPACK=OFF -DUSE_REPLXX=OFF -DENABLE_CLICKHOUSE_ALL=OFF
+ -DENABLE_GWP_ASAN=OFF -DENABLE_NUMACTL=OFF -DENABLE_GOOGLE_CLOUD_CPP=OFF
+ -DENABLE_ARROW_FLIGHT=OFF -DENABLE_XRAY=0 -DUSE_SYSTEM_COMPILER_RT=OFF
+ "-DCOMPILER_FLAGS=-fvisibility=hidden -fvisibility-inlines-hidden"
+ COMMAND ${CMAKE_COMMAND} --build ${CH_BINARY_DIR} --target cxx
+ COMMAND
+ ${CMAKE_COMMAND}
+ -DGLUTEN_PAIMON_LIBCXX_INCLUDE=${GLUTEN_PAIMON_LIBCXX_INCLUDE}
+
-DGLUTEN_PAIMON_LIBCXX_SOURCE_INCLUDE=${GLUTEN_PAIMON_LIBCXX_SOURCE_INCLUDE}
+ -DGLUTEN_PAIMON_LIBCXXABI_INCLUDE=${GLUTEN_PAIMON_LIBCXXABI_INCLUDE}
+ -DGLUTEN_PAIMON_LIBCXX_ARCHIVE=${GLUTEN_PAIMON_LIBCXX_ARCHIVE}
+ -DGLUTEN_PAIMON_LIBCXXABI_ARCHIVE=${GLUTEN_PAIMON_LIBCXXABI_ARCHIVE}
+ -DGLUTEN_PAIMON_BUILTINS_ARCHIVE=${GLUTEN_PAIMON_BUILTINS_ARCHIVE}
+ -DGLUTEN_PAIMON_LIBUNWIND_ARCHIVE=${GLUTEN_PAIMON_LIBUNWIND_ARCHIVE} -P
+ ${CMAKE_CURRENT_SOURCE_DIR}/cmake/CheckPaimonLibcxx.cmake
+ USES_TERMINAL VERBATIM)
+
+file(MAKE_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/build/deps")
+
+ExternalProject_Add(
+ paimon_cpp
+ DEPENDS prepare_paimon_libcxx
+ GIT_REPOSITORY ${PAIMON_GIT_REPOSITORY}
+ GIT_TAG ${PAIMON_GIT_TAG}
+ GIT_SHALLOW TRUE
+ GIT_PROGRESS TRUE
+ SOURCE_DIR ${PAIMON_EP_SOURCE_DIR}
+ BINARY_DIR ${PAIMON_EP_BINARY_DIR}
+ CMAKE_ARGS
+ -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
+ -DCMAKE_INSTALL_PREFIX=${PAIMON_INSTALL_PREFIX}
+ -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}
+ -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
+ -DCMAKE_SYSROOT=${GLUTEN_PAIMON_SYSROOT}
+ -DCMAKE_C_FLAGS=${GLUTEN_PAIMON_SYSROOT_FLAGS}
+ -DCMAKE_CXX_FLAGS=${GLUTEN_PAIMON_CXX_FLAGS}
+ -DCMAKE_CXX_STANDARD_LIBRARIES=${GLUTEN_PAIMON_CXX_STANDARD_LIBRARIES}
+ -DGLUTEN_PAIMON_LIBCXX_ARCHIVE=${GLUTEN_PAIMON_LIBCXX_ARCHIVE}
+ -DGLUTEN_PAIMON_LIBCXXABI_ARCHIVE=${GLUTEN_PAIMON_LIBCXXABI_ARCHIVE}
+ -DGLUTEN_PAIMON_LIBUNWIND_ARCHIVE=${GLUTEN_PAIMON_LIBUNWIND_ARCHIVE}
+
-DGLUTEN_PAIMON_CXX_STANDARD_LIBRARIES=${GLUTEN_PAIMON_CXX_STANDARD_LIBRARIES}
+
-DGLUTEN_PAIMON_COMPAT_SOURCE=${CMAKE_CURRENT_SOURCE_DIR}/cmake/PaimonGlibcCompat.c
+ -DLIBUNWIND_LIBRARY=${GLUTEN_PAIMON_LIBUNWIND_ARCHIVE}
+ -DCMAKE_PREFIX_PATH=
+ -DCMAKE_FIND_USE_SYSTEM_PACKAGE_REGISTRY=OFF
+ -DCMAKE_FIND_USE_PACKAGE_REGISTRY=OFF
+ -DPAIMON_BUILD_TESTS=OFF
+ -DPAIMON_BUILD_SHARED=ON
+ -DPAIMON_DEPENDENCY_SOURCE=BUNDLED
+ -Dfmt_SOURCE=BUNDLED
+ -Dglog_SOURCE=BUNDLED
+ -DRapidJSON_SOURCE=BUNDLED
+ -DCMAKE_IGNORE_PATH=${CMAKE_CURRENT_SOURCE_DIR}/../ep
+ PATCH_COMMAND
+ ${CMAKE_COMMAND} -DPAIMON_SOURCE_DIR=<SOURCE_DIR>
+ -DGLUTEN_PAIMON_LIBCXX_ARCHIVE=${GLUTEN_PAIMON_LIBCXX_ARCHIVE}
+ -DGLUTEN_PAIMON_LIBCXXABI_ARCHIVE=${GLUTEN_PAIMON_LIBCXXABI_ARCHIVE}
+ -DGLUTEN_PAIMON_LIBUNWIND_ARCHIVE=${GLUTEN_PAIMON_LIBUNWIND_ARCHIVE}
+
-DGLUTEN_PAIMON_CXX_STANDARD_LIBRARIES=${GLUTEN_PAIMON_CXX_STANDARD_LIBRARIES}
+
-DGLUTEN_PAIMON_COMPAT_SOURCE=${CMAKE_CURRENT_SOURCE_DIR}/cmake/PaimonGlibcCompat.c
+ -P ${CMAKE_CURRENT_SOURCE_DIR}/cmake/PatchPaimonArrowThrift.cmake
+ BUILD_COMMAND ${CMAKE_COMMAND} --build <BINARY_DIR> --parallel
+ BUILD_ALWAYS TRUE
+ INSTALL_COMMAND ${CMAKE_COMMAND} --build <BINARY_DIR> --target install
+ USES_TERMINAL_DOWNLOAD ON
+ USES_TERMINAL_BUILD ON
+ UPDATE_DISCONNECTED TRUE)
+
+# Paimon's object libraries include headers installed by these ExternalProject
+# targets, but its build graph does not wait for their install steps. Build the
+# bundled dependencies before compiling Paimon in a parallel build.
+ExternalProject_Add_Step(
+ paimon_cpp prepare_dependencies
+ COMMAND ${CMAKE_COMMAND} --build <BINARY_DIR> --target arrow_ep avro_ep
fmt_ep
+ glog_ep orc_ep protobuf_ep rapidjson_ep tbb_ep --parallel
+ DEPENDEES configure
+ DEPENDERS build
+ USES_TERMINAL TRUE)
+
+set(GLUTEN_PAIMON_CMAKE_DIR "${PAIMON_INSTALL_PREFIX}/lib/cmake/Paimon")
+
+# Register after the caller defines the libch build command and output
+# directory.
+function(gluten_bundle_paimon_libs libch_output_dir deployment_dir)
+ # Ship the combined Paimon shared library next to libch.so for deployment.
+ add_custom_target(
+ bundle_paimon_libs
+ COMMAND ${CMAKE_COMMAND} -E make_directory ${libch_output_dir}
+ COMMAND
+ ${CMAKE_COMMAND} -E copy_if_different
+ ${PAIMON_INSTALL_PREFIX}/lib/libpaimon.so
${libch_output_dir}/libpaimon.so
+ # Jenkins removes cpp-ch after copying libch.so to the repository root.
+ # Preserve both runtime libraries there before the build tree is removed.
+ COMMAND ${CMAKE_COMMAND} -E copy_if_different ${libch_output_dir}/libch.so
+ ${deployment_dir}/libch.so
+ COMMAND ${CMAKE_COMMAND} -E copy_if_different
+ ${libch_output_dir}/libpaimon.so ${deployment_dir}/libpaimon.so
+ DEPENDS _build_ch
+ COMMENT "Copy libch.so and libpaimon.so for deployment"
+ VERBATIM)
+endfunction()
diff --git a/cpp-ch/cmake/CheckPaimonLibcxx.cmake
b/cpp-ch/cmake/CheckPaimonLibcxx.cmake
new file mode 100644
index 0000000000..e28ab378f8
--- /dev/null
+++ b/cpp-ch/cmake/CheckPaimonLibcxx.cmake
@@ -0,0 +1,34 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
contributor
+# license agreements. See the NOTICE file distributed with this work for
+# additional information regarding copyright ownership. The ASF licenses this
+# file to You under the Apache License, Version 2.0 (the "License"); you may
not
+# use this file except in compliance with the License. You may obtain a copy
of
+# the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+foreach(
+ _header IN
+ ITEMS "${GLUTEN_PAIMON_LIBCXX_INCLUDE}/__config_site"
+ "${GLUTEN_PAIMON_LIBCXX_SOURCE_INCLUDE}/string"
+ "${GLUTEN_PAIMON_LIBCXXABI_INCLUDE}/cxxabi.h")
+ if(NOT EXISTS "${_header}")
+ message(FATAL_ERROR "ClickHouse C++ header was not generated: ${_header}")
+ endif()
+endforeach()
+
+foreach(
+ _archive IN
+ ITEMS "${GLUTEN_PAIMON_LIBCXX_ARCHIVE}" "${GLUTEN_PAIMON_LIBCXXABI_ARCHIVE}"
+ "${GLUTEN_PAIMON_LIBUNWIND_ARCHIVE}"
+ "${GLUTEN_PAIMON_BUILTINS_ARCHIVE}")
+ if(NOT EXISTS "${_archive}")
+ message(FATAL_ERROR "ClickHouse runtime archive was not built:
${_archive}")
+ endif()
+endforeach()
diff --git a/cpp-ch/cmake/PaimonGlibcCompat.c b/cpp-ch/cmake/PaimonGlibcCompat.c
new file mode 100644
index 0000000000..ba7404c819
--- /dev/null
+++ b/cpp-ch/cmake/PaimonGlibcCompat.c
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/* Link-time wrappers for selected glibc calls in libpaimon.so. They bind math
+ * calls to older symbol versions and emulate pthread_cond_clockwait on older
+ * glibc releases; the getenv wrapper avoids a Bash startup symbol conflict. */
+#define _GNU_SOURCE
+
+#include <errno.h>
+#include <math.h>
+#include <pthread.h>
+#include <stdlib.h>
+#include <time.h>
+
+/* Bash exports its own getenv, which is unsafe before Bash has initialized. */
+char *__wrap_getenv(const char *name) { return secure_getenv(name); }
+
+/* glibc 2.27/2.29 changed the default versions of these functions. */
+#define WRAP_UNARY(name, type) \
+ extern type old_##name(type); \
+ __asm__(".symver old_" #name "," #name "@GLIBC_2.2.5"); \
+ type __wrap_##name(type value) { return old_##name(value); }
+
+#define WRAP_BINARY(name, type) \
+ extern type old_##name(type, type); \
+ __asm__(".symver old_" #name "," #name "@GLIBC_2.2.5"); \
+ type __wrap_##name(type left, type right) { return old_##name(left,
right); }
+
+WRAP_UNARY(exp, double)
+WRAP_UNARY(exp2, double)
+WRAP_UNARY(expf, float)
+WRAP_UNARY(log, double)
+WRAP_UNARY(log2, double)
+WRAP_UNARY(log2f, float)
+WRAP_UNARY(logf, float)
+WRAP_BINARY(pow, double)
+WRAP_BINARY(powf, float)
+
+/* pthread_cond_clockwait was introduced in glibc 2.30. */
+int __wrap_pthread_cond_clockwait(
+ pthread_cond_t *cond, pthread_mutex_t *mutex, clockid_t clock_id,
+ const struct timespec *deadline)
+{
+ if (clock_id == CLOCK_REALTIME)
+ return pthread_cond_timedwait(cond, mutex, deadline);
+ if (clock_id != CLOCK_MONOTONIC || !deadline || deadline->tv_nsec < 0 ||
+ deadline->tv_nsec >= 1000000000L)
+ return EINVAL;
+
+ struct timespec now_mono;
+ struct timespec now_real;
+ if (clock_gettime(CLOCK_MONOTONIC, &now_mono) != 0 ||
+ clock_gettime(CLOCK_REALTIME, &now_real) != 0)
+ return errno;
+
+ time_t seconds = deadline->tv_sec - now_mono.tv_sec;
+ long nanoseconds = deadline->tv_nsec - now_mono.tv_nsec;
+ if (nanoseconds < 0) {
+ --seconds;
+ nanoseconds += 1000000000L;
+ }
+ if (seconds < 0)
+ return ETIMEDOUT;
+
+ struct timespec real_deadline = {now_real.tv_sec + seconds,
+ now_real.tv_nsec + nanoseconds};
+ if (real_deadline.tv_nsec >= 1000000000L) {
+ ++real_deadline.tv_sec;
+ real_deadline.tv_nsec -= 1000000000L;
+ }
+ return pthread_cond_timedwait(cond, mutex, &real_deadline);
+}
diff --git a/cpp-ch/cmake/PatchPaimonArrowThrift.cmake
b/cpp-ch/cmake/PatchPaimonArrowThrift.cmake
new file mode 100644
index 0000000000..7d36e566c3
--- /dev/null
+++ b/cpp-ch/cmake/PatchPaimonArrowThrift.cmake
@@ -0,0 +1,284 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Adapt upstream paimon-cpp's CMake files before configuring its build. Bundle
+# its dependencies and plugins into libpaimon.so, link ClickHouse's libc++
+# runtime, and add selected glibc compatibility wrappers. Paimon keeps its
+# upstream Arrow version despite this script's historical name.
+set(_toolchain "${PAIMON_SOURCE_DIR}/cmake_modules/ThirdpartyToolchain.cmake")
+file(READ "${_toolchain}" _contents)
+set(_original_contents "${_contents}")
+if(NOT _contents MATCHES "-DThrift_SOURCE=BUNDLED")
+ set(_anchor " -DARROW_DEPENDENCY_USE_SHARED=OFF")
+ set(_replacement " -DThrift_SOURCE=BUNDLED\n${_anchor}")
+ string(FIND "${_contents}" "${_anchor}" _position)
+ if(_position EQUAL -1)
+ message(
+ FATAL_ERROR
+ "Cannot enable bundled Thrift in ${_toolchain}: Arrow CMake arguments
changed"
+ )
+ endif()
+ string(REPLACE "${_anchor}" "${_replacement}" _contents "${_contents}")
+endif()
+if(NOT _contents MATCHES "-DWITH_UNWIND=none[ \n]+-DWITH_GFLAGS=OFF")
+ set(_anchor " -DWITH_GFLAGS=OFF")
+ set(_replacement " -DWITH_UNWIND=none\n${_anchor}")
+ string(FIND "${_contents}" "${_anchor}" _position)
+ if(_position EQUAL -1)
+ message(
+ FATAL_ERROR
+ "Cannot disable glog libunwind in ${_toolchain}: glog CMake arguments
changed"
+ )
+ endif()
+ string(REPLACE "${_anchor}" "${_replacement}" _contents "${_contents}")
+endif()
+if(_contents MATCHES "GLUTEN_PAIMON_EXE_LINKER_FLAGS")
+ string(
+ REPLACE
+ "\"-DCMAKE_EXE_LINKER_FLAGS=\${GLUTEN_PAIMON_EXE_LINKER_FLAGS}\""
+
"\"-DCMAKE_CXX_STANDARD_LIBRARIES=\${GLUTEN_PAIMON_CXX_STANDARD_LIBRARIES}\""
+ _contents
+ "${_contents}")
+elseif(NOT _contents MATCHES "GLUTEN_PAIMON_CXX_STANDARD_LIBRARIES")
+ set(_anchor " -DCMAKE_CXX_FLAGS=\${EP_CXX_FLAGS}")
+ set(_replacement
+ "${_anchor}\n
\"-DCMAKE_CXX_STANDARD_LIBRARIES=\${GLUTEN_PAIMON_CXX_STANDARD_LIBRARIES}\""
+ )
+ string(FIND "${_contents}" "${_anchor}" _position)
+ if(_position EQUAL -1)
+ message(
+ FATAL_ERROR
+ "Cannot propagate ClickHouse libc++ linker flags: ${_toolchain}
changed"
+ )
+ endif()
+ string(REPLACE "${_anchor}" "${_replacement}" _contents "${_contents}")
+endif()
+string(
+ FIND "${_contents}"
+
"\"-DCMAKE_CXX_STANDARD_LIBRARIES=\${GLUTEN_PAIMON_CXX_STANDARD_LIBRARIES}\"\n
-DCMAKE_EXE_LINKER_FLAGS="
+ _linker_flags_position)
+if(_linker_flags_position EQUAL -1)
+ string(
+ REPLACE
+
"\"-DCMAKE_CXX_STANDARD_LIBRARIES=\${GLUTEN_PAIMON_CXX_STANDARD_LIBRARIES}\""
+
"\"-DCMAKE_CXX_STANDARD_LIBRARIES=\${GLUTEN_PAIMON_CXX_STANDARD_LIBRARIES}\"\n
-DCMAKE_EXE_LINKER_FLAGS="
+ _contents
+ "${_contents}")
+endif()
+if(NOT _contents STREQUAL _original_contents)
+ file(WRITE "${_toolchain}" "${_contents}")
+endif()
+
+# Keep the main Paimon shared library and fold its plugin object files into it.
+# The factories register themselves through static initializers, so the whole
+# object libraries must be included even if no symbols are referenced directly.
+set(_build_utils "${PAIMON_SOURCE_DIR}/cmake_modules/BuildUtils.cmake")
+file(READ "${_build_utils}" _contents)
+set(_original_contents "${_contents}")
+set(_dependency_anchor " if(TARGET \${_paimon_mapped_dep})")
+set(_dependency_replacement
+ " if(TARGET \${_paimon_mapped_dep} AND NOT (NOT LIB_NAME
STREQUAL \"paimon\" AND _paimon_mapped_dep STREQUAL \"paimon_shared\"))"
+)
+if(NOT _contents MATCHES "_paimon_mapped_dep STREQUAL \\\"paimon_shared\\\"")
+ string(FIND "${_contents}" "${_dependency_anchor}" _position)
+ if(_position EQUAL -1)
+ message(
+ FATAL_ERROR
+ "Cannot remove Paimon plugin dependency cycle: ${_build_utils}
changed")
+ endif()
+ string(REPLACE "${_dependency_anchor}" "${_dependency_replacement}" _contents
+ "${_contents}")
+endif()
+set(_anchor " if(ARG_BUILD_STATIC)\n")
+set(_replacement
+ " if(NOT LIB_NAME STREQUAL \"paimon\")\n set(BUILD_SHARED OFF)\n
endif()\n${_anchor}"
+)
+if(NOT _contents MATCHES "if\\(NOT LIB_NAME STREQUAL \\\"paimon\\\"\\)")
+ string(FIND "${_contents}" "${_anchor}" _position)
+ if(_position EQUAL -1)
+ message(
+ FATAL_ERROR
+ "Cannot configure a single Paimon shared library: ${_build_utils}
changed"
+ )
+ endif()
+ string(REPLACE "${_anchor}" "${_replacement}" _contents "${_contents}")
+endif()
+if(NOT _contents STREQUAL _original_contents)
+ file(WRITE "${_build_utils}" "${_contents}")
+endif()
+
+set(_root_cmake "${PAIMON_SOURCE_DIR}/CMakeLists.txt")
+file(READ "${_root_cmake}" _contents)
+set(_original_contents "${_contents}")
+set(_anchor "add_subdirectory(test/inte)")
+set(_replacement
+ [=[
+add_subdirectory(test/inte)
+
+# Gluten deploys one Paimon DSO. Include every enabled plugin's object files
+# in the main library, preserving their static factory registrations.
+foreach(_plugin IN ITEMS
+ paimon_file_index paimon_global_index paimon_local_file_system
+ paimon_blob_file_format paimon_orc_file_format
+ paimon_parquet_file_format paimon_avro_file_format
+ paimon_s3_file_system paimon_jindo_file_system
+ paimon_lucene_index paimon_lumina_index paimon_tantivy_index)
+ if(TARGET ${_plugin}_objlib)
+ target_sources(paimon_shared PRIVATE
$<TARGET_OBJECTS:${_plugin}_objlib>)
+ endif()
+endforeach()
+target_link_libraries(paimon_shared PRIVATE parquet)
+if(PAIMON_ENABLE_ORC)
+ target_link_libraries(paimon_shared PRIVATE orc::orc)
+endif()
+if(PAIMON_ENABLE_AVRO)
+ target_link_libraries(paimon_shared PRIVATE avro)
+endif()
+# Keep the C++ runtime self-contained; executors need only libpaimon.so.
+target_link_options(paimon_shared PRIVATE -nostdlib++)
+target_link_libraries(paimon_shared PRIVATE -Wl,--whole-archive
+ "${GLUTEN_PAIMON_LIBCXX_ARCHIVE}"
+ -Wl,--no-whole-archive
+ "${GLUTEN_PAIMON_LIBCXXABI_ARCHIVE}"
+ "${GLUTEN_PAIMON_LIBUNWIND_ARCHIVE}")
+]=])
+if(NOT _contents MATCHES "Gluten deploys one Paimon DSO")
+ string(FIND "${_contents}" "${_anchor}" _position)
+ if(_position EQUAL -1)
+ message(
+ FATAL_ERROR
+ "Cannot add Paimon plugins to the main library: ${_root_cmake}
changed")
+ endif()
+ string(REPLACE "${_anchor}" "${_replacement}" _contents "${_contents}")
+endif()
+if(NOT _contents MATCHES
+ "target_link_options\\(paimon_shared PRIVATE -nostdlib\\+\\+\\)")
+ if(NOT GLUTEN_PAIMON_LIBCXX_ARCHIVE
+ OR NOT GLUTEN_PAIMON_LIBCXXABI_ARCHIVE
+ OR NOT GLUTEN_PAIMON_LIBUNWIND_ARCHIVE)
+ message(
+ FATAL_ERROR "ClickHouse C++ runtime archives are required to link
Paimon")
+ endif()
+ set(_anchor "target_link_libraries(paimon_shared PRIVATE avro)\nendif()")
+ set(_replacement
+ "${_anchor}\n# Keep the C++ runtime self-contained; executors need only
libpaimon.so.\ntarget_link_options(paimon_shared PRIVATE
-nostdlib++)\ntarget_link_libraries(paimon_shared PRIVATE -Wl,--whole-archive\n
\"\${GLUTEN_PAIMON_LIBCXX_ARCHIVE}\"\n -Wl,--no-whole-archive\n
\"\${GLUTEN_PAIMON_LIBCXXABI_ARCHIVE}\"\n
\"\${GLUTEN_PAIMON_LIBUNWIND_ARCHIVE}\")"
+ )
+ string(FIND "${_contents}" "${_anchor}" _position)
+ if(_position EQUAL -1)
+ message(
+ FATAL_ERROR
+ "Cannot link LLVM C++ runtime into ${_root_cmake}: Paimon CMake
changed"
+ )
+ endif()
+ string(REPLACE "${_anchor}" "${_replacement}" _contents "${_contents}")
+endif()
+# Bind math calls to their original glibc versions and emulate the newer
pthread
+# clock wait call, including references from bundled static archives.
+if(NOT GLUTEN_PAIMON_COMPAT_SOURCE OR NOT EXISTS
+ "${GLUTEN_PAIMON_COMPAT_SOURCE}")
+ message(
+ FATAL_ERROR "GLUTEN_PAIMON_COMPAT_SOURCE must name the compatibility
source"
+ )
+endif()
+set(_compat_anchor "target_link_options(paimon_shared PRIVATE -nostdlib++)")
+set(_compat_symbols
+ exp
+ exp2
+ expf
+ getenv
+ log
+ log2
+ log2f
+ logf
+ pow
+ powf
+ pthread_cond_clockwait)
+if(NOT _contents MATCHES "GLUTEN_PAIMON_COMPAT_SOURCE")
+ set(_compat_options "")
+ foreach(_symbol IN LISTS _compat_symbols)
+ string(APPEND _compat_options " -Wl,--wrap=${_symbol}")
+ endforeach()
+ set(_compat_replacement
+ "${_compat_anchor}\ntarget_sources(paimon_shared PRIVATE
\"\${GLUTEN_PAIMON_COMPAT_SOURCE}\")\ntarget_link_options(paimon_shared
PRIVATE${_compat_options})"
+ )
+ string(FIND "${_contents}" "${_compat_anchor}" _compat_position)
+ if(_compat_position EQUAL -1)
+ message(
+ FATAL_ERROR
+ "Cannot add glibc compatibility to ${_root_cmake}: Paimon CMake
changed"
+ )
+ endif()
+ string(REPLACE "${_compat_anchor}" "${_compat_replacement}" _contents
+ "${_contents}")
+endif()
+# Also update a source tree patched by an earlier version of this script.
+foreach(_symbol IN LISTS _compat_symbols)
+ if(NOT _contents MATCHES "--wrap=${_symbol}([ )]|$)")
+ set(_anchor " -Wl,--wrap=exp")
+ string(FIND "${_contents}" "${_anchor}" _position)
+ if(_position EQUAL -1)
+ message(
+ FATAL_ERROR
+ "Cannot wrap ${_symbol}: Paimon link options changed in
${_root_cmake}"
+ )
+ endif()
+ string(REPLACE "${_anchor}" " -Wl,--wrap=${_symbol}${_anchor}" _contents
+ "${_contents}")
+ endif()
+endforeach()
+foreach(
+ _symbol IN
+ ITEMS malloc
+ free
+ calloc
+ realloc
+ memalign
+ aligned_alloc
+ posix_memalign)
+ string(REPLACE " -Wl,--wrap=${_symbol}" "" _contents "${_contents}")
+endforeach()
+string(REGEX
+ REPLACE "\"[^\"]*/libc\\+\\+\\.a\""
+ "\"\${GLUTEN_PAIMON_LIBCXX_ARCHIVE}\"" _contents "${_contents}")
+string(
+ REGEX
+ REPLACE "\"[^\"]*/libc\\+\\+abi\\.a\""
+ "\"\${GLUTEN_PAIMON_LIBCXXABI_ARCHIVE}\"" _contents "${_contents}")
+string(
+ REGEX
+ REPLACE "\"[^\"]*/libunwind\\.a\"" "\"\${GLUTEN_PAIMON_LIBUNWIND_ARCHIVE}\""
+ _contents "${_contents}")
+if(NOT _contents STREQUAL _original_contents)
+ file(WRITE "${_root_cmake}" "${_contents}")
+endif()
+# These two implementation files are already compiled into paimon_objlib.
+set(_global_index_cmake
+ "${PAIMON_SOURCE_DIR}/src/paimon/common/global_index/CMakeLists.txt")
+file(READ "${_global_index_cmake}" _contents)
+set(_anchor "add_paimon_lib(paimon_global_index")
+set(_replacement
+ "list(REMOVE_ITEM PAIMON_GLOBAL_INDEX_SRC\n
offset_global_index_reader.cpp union_global_index_reader.cpp)\n\n${_anchor}"
+)
+if(NOT _contents MATCHES "list\\(REMOVE_ITEM PAIMON_GLOBAL_INDEX_SRC")
+ string(FIND "${_contents}" "${_anchor}" _position)
+ if(_position EQUAL -1)
+ message(
+ FATAL_ERROR
+ "Cannot deduplicate Paimon index sources: ${_global_index_cmake}
changed"
+ )
+ endif()
+ string(REPLACE "${_anchor}" "${_replacement}" _contents "${_contents}")
+ file(WRITE "${_global_index_cmake}" "${_contents}")
+endif()
diff --git a/cpp-ch/local-engine/CMakeLists.txt
b/cpp-ch/local-engine/CMakeLists.txt
index 1955d10bc6..3190e1f52a 100644
--- a/cpp-ch/local-engine/CMakeLists.txt
+++ b/cpp-ch/local-engine/CMakeLists.txt
@@ -202,6 +202,19 @@ endmacro()
add_memory_tracking_wrappers(${LOCALENGINE_SHARED_LIB})
+if(GLUTEN_ENABLE_NATIVE_PAIMON AND NOT APPLE)
+ # Keep the Paimon dependency discoverable after relocating the runtime
+ # libraries.
+ set_property(
+ TARGET ${LOCALENGINE_SHARED_LIB}
+ APPEND
+ PROPERTY BUILD_RPATH "$ORIGIN")
+ set_property(
+ TARGET ${LOCALENGINE_SHARED_LIB}
+ APPEND
+ PROPERTY INSTALL_RPATH "$ORIGIN")
+endif()
+
if(NOT APPLE)
if(ENABLE_JEMALLOC)
target_link_options(
diff --git a/cpp-ch/local-engine/Storages/SubstraitSource/CMakeLists.txt
b/cpp-ch/local-engine/Storages/SubstraitSource/CMakeLists.txt
index 639c1f043f..baf1227a21 100644
--- a/cpp-ch/local-engine/Storages/SubstraitSource/CMakeLists.txt
+++ b/cpp-ch/local-engine/Storages/SubstraitSource/CMakeLists.txt
@@ -15,12 +15,32 @@
set(ARROW_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/arrow/cpp/src")
+option(GLUTEN_ENABLE_NATIVE_PAIMON "Enable Paimon native read in libch" ON)
+
add_headers_and_sources(substrait_source .)
add_headers_and_sources(substrait_source Delta)
add_headers_and_sources(substrait_source Delta/Bitmap)
add_headers_and_sources(substrait_source Iceberg)
+if(GLUTEN_ENABLE_NATIVE_PAIMON)
+ add_headers_and_sources(substrait_source Paimon)
+endif()
add_library(substrait_source ${substrait_source_sources})
+
+if(GLUTEN_ENABLE_NATIVE_PAIMON)
+ find_package(Paimon 0.3.0 CONFIG REQUIRED)
+ if(NOT PAIMON_BUNDLED_DEPS_DIR)
+ message(FATAL_ERROR "PAIMON_BUNDLED_DEPS_DIR is required for native
Paimon")
+ endif()
+ target_compile_definitions(substrait_source
+ PUBLIC GLUTEN_ENABLE_NATIVE_PAIMON=1)
+ target_link_libraries(substrait_source PUBLIC Paimon::paimon_shared)
+ target_link_options(substrait_source PUBLIC "-Wl,-rpath,\$ORIGIN")
+else()
+ target_compile_definitions(substrait_source
+ PUBLIC GLUTEN_ENABLE_NATIVE_PAIMON=0)
+endif()
+
target_compile_options(
substrait_source PRIVATE -Wno-suggest-destructor-override
-Wno-inconsistent-missing-destructor-override)
diff --git a/cpp-ch/local-engine/Storages/SubstraitSource/FileReader.cpp
b/cpp-ch/local-engine/Storages/SubstraitSource/FileReader.cpp
index a3ee72d409..6f027b68d4 100644
--- a/cpp-ch/local-engine/Storages/SubstraitSource/FileReader.cpp
+++ b/cpp-ch/local-engine/Storages/SubstraitSource/FileReader.cpp
@@ -290,7 +290,7 @@ std::unique_ptr<NormalFileReader> createNormalFileReader(
return nullptr;
// when there is a '__delta_internal_is_row_deleted' column, it needs to
use DeltaReader to read data and add column
- if (DeltaVirtualMeta::hasMetaColumns(to_read_header_))
+ if (!file->getFileInfo().has_paimon() &&
DeltaVirtualMeta::hasMetaColumns(to_read_header_))
{
String row_index_ids_encoded;
String row_index_filter_type;
@@ -336,4 +336,4 @@ std::unique_ptr<BaseReader> BaseReader::create(
}
-}
\ No newline at end of file
+}
diff --git a/cpp-ch/local-engine/Storages/SubstraitSource/FormatFile.cpp
b/cpp-ch/local-engine/Storages/SubstraitSource/FormatFile.cpp
index 6829d34d4a..5cc6a07e2b 100644
--- a/cpp-ch/local-engine/Storages/SubstraitSource/FormatFile.cpp
+++ b/cpp-ch/local-engine/Storages/SubstraitSource/FormatFile.cpp
@@ -24,6 +24,9 @@
#include <Common/GlutenStringUtils.h>
#include <Common/logger_useful.h>
+#if GLUTEN_ENABLE_NATIVE_PAIMON
+#include <Storages/SubstraitSource/Paimon/PaimonFormatFile.h>
+#endif
#if USE_PARQUET
#include <Storages/SubstraitSource/ParquetFormatFile.h>
@@ -143,6 +146,16 @@ FormatFile::FormatFile(DB::ContextPtr context_, const
SubstraitInputFile & file_
FormatFilePtr FormatFileUtil::createFile(
DB::ContextPtr context, ReadBufferBuilderPtr read_buffer_builder, const
substrait::ReadRel::LocalFiles::FileOrFiles & file)
{
+ if (file.has_paimon())
+ {
+#if GLUTEN_ENABLE_NATIVE_PAIMON
+ return std::make_shared<PaimonFormatFile>(context, file,
read_buffer_builder);
+#else
+ throw DB::Exception(
+ DB::ErrorCodes::NOT_IMPLEMENTED,
+ "Paimon native read is disabled in this libch build
(GLUTEN_ENABLE_NATIVE_PAIMON=OFF)");
+#endif
+ }
#if USE_PARQUET
if (file.has_parquet() || (file.has_iceberg() &&
file.iceberg().has_parquet()))
{
diff --git
a/cpp-ch/local-engine/Storages/SubstraitSource/Paimon/HdfsPaimonFileSystem.cpp
b/cpp-ch/local-engine/Storages/SubstraitSource/Paimon/HdfsPaimonFileSystem.cpp
new file mode 100644
index 0000000000..909be91cbb
--- /dev/null
+++
b/cpp-ch/local-engine/Storages/SubstraitSource/Paimon/HdfsPaimonFileSystem.cpp
@@ -0,0 +1,290 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "HdfsPaimonFileSystem.h"
+
+#include <Interpreters/Context.h>
+#include <Storages/ObjectStorage/HDFS/HDFSCommon.h>
+#include <Common/logger_useful.h>
+#include <Poco/URI.h>
+#include <hdfs/hdfs.h>
+
+#include <algorithm>
+#include <cerrno>
+#include <chrono>
+#include <limits>
+#include <mutex>
+#include <utility>
+
+namespace local_engine::paimon
+{
+namespace
+{
+using Status = ::paimon::Status;
+
+struct HdfsConnection
+{
+ DB::HDFSFSPtr fs;
+ explicit HdfsConnection(const std::string & uri, DB::ContextPtr context)
+ {
+ auto builder = DB::createHDFSBuilder(uri, context->getConfigRef());
+ fs = DB::createHDFSFS(builder.get());
+ }
+};
+
+std::string pathPart(const std::string & uri)
+{
+ Poco::URI parsed(uri);
+ return parsed.getPath();
+}
+
+Status hdfsError(const std::string & operation, const std::string & path)
+{
+ return Status::IOError(operation, " ", path, ": ", hdfsGetLastError());
+}
+
+class HdfsStatus final : public ::paimon::FileStatus, public
::paimon::BasicFileStatus
+{
+ std::string path_;
+ int64_t size_;
+ int64_t modification_time_;
+ bool directory_;
+
+public:
+ explicit HdfsStatus(const hdfsFileInfo & info)
+ : path_(info.mName)
+ , size_(info.mSize)
+ , modification_time_(static_cast<int64_t>(info.mLastMod) * 1000)
+ , directory_(info.mKind == kObjectKindDirectory)
+ {
+ }
+
+ int64_t GetLen() const override { return size_; }
+ bool IsDir() const override { return directory_; }
+ std::string GetPath() const override { return path_; }
+ int64_t GetModificationTime() const override { return modification_time_; }
+};
+
+class HdfsInputStream final : public ::paimon::InputStream
+{
+ std::shared_ptr<HdfsConnection> connection_;
+ hdfsFile file_;
+ std::string uri_;
+ int64_t length_;
+ int64_t position_ = 0;
+ uint64_t read_calls_ = 0;
+ uint64_t read_bytes_ = 0;
+ uint64_t read_time_ns_ = 0;
+ mutable std::mutex mutex_;
+
+public:
+ HdfsInputStream(std::shared_ptr<HdfsConnection> connection, hdfsFile file,
std::string uri, int64_t length)
+ : connection_(std::move(connection)), file_(file),
uri_(std::move(uri)), length_(length) { }
+ ~HdfsInputStream() override
+ {
+ Close();
+ if (read_calls_ > 0)
+ LOG_INFO(
+ &Poco::Logger::get("PaimonHDFS"),
+ "HDFS read summary: path={}, calls={}, bytes={},
read_time_ms={}",
+ uri_,
+ read_calls_,
+ read_bytes_,
+ read_time_ns_ / 1000000);
+ }
+
+ Status Close() override
+ {
+ std::lock_guard lock(mutex_);
+ if (file_ && hdfsCloseFile(connection_->fs.get(), file_) != 0)
+ return hdfsError("close", uri_);
+ file_ = nullptr;
+ return Status::OK();
+ }
+
+ Status Seek(int64_t offset, ::paimon::SeekOrigin origin) override
+ {
+ std::lock_guard lock(mutex_);
+ if (origin != ::paimon::FS_SEEK_SET && origin != ::paimon::FS_SEEK_CUR
&& origin != ::paimon::FS_SEEK_END)
+ return Status::Invalid("Invalid HDFS seek origin: ", uri_);
+ int64_t base = origin == ::paimon::FS_SEEK_END ? length_ : origin ==
::paimon::FS_SEEK_CUR ? position_ : 0;
+ if (offset < -base || offset > length_ - base)
+ return Status::Invalid("HDFS seek out of range: ", uri_);
+ position_ = base + offset;
+ return Status::OK();
+ }
+
+ ::paimon::Result<int64_t> GetPos() const override
+ {
+ std::lock_guard lock(mutex_);
+ return position_;
+ }
+
+ ::paimon::Result<int64_t> Read(char * buffer, int64_t size) override
+ {
+ std::lock_guard lock(mutex_);
+ auto result = readAt(buffer, size, position_);
+ if (result.ok())
+ position_ += result.value();
+ return result;
+ }
+
+ ::paimon::Result<int64_t> Read(char * buffer, int64_t size, int64_t
offset) override
+ {
+ std::lock_guard lock(mutex_);
+ return readAt(buffer, size, offset);
+ }
+
+ void ReadAsync(char * buffer, int64_t size, int64_t offset,
std::function<void(Status)> && callback) override
+ {
+ auto result = Read(buffer, size, offset);
+ callback(result.ok() ? Status::OK() : result.status());
+ }
+
+ ::paimon::Result<std::string> GetUri() const override { return uri_; }
+ ::paimon::Result<int64_t> Length() const override { return length_; }
+
+private:
+ ::paimon::Result<int64_t> readAt(char * buffer, int64_t size, int64_t
offset)
+ {
+ if (!file_ || size < 0 || offset < 0)
+ return Status::Invalid("Invalid HDFS read: ", uri_);
+ if (offset > length_ || size > length_ - offset)
+ return Status::IOError("HDFS read exceeds file length: ", uri_);
+ int64_t total = 0;
+ while (total < size)
+ {
+ auto count = static_cast<tSize>(std::min<int64_t>(size - total,
std::numeric_limits<tSize>::max()));
+ auto start = std::chrono::steady_clock::now();
+ auto read = hdfsPread(connection_->fs.get(), file_, buffer +
total, count, offset + total);
+ read_time_ns_ +=
std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::steady_clock::now()
- start).count();
+ ++read_calls_;
+ if (read < 0)
+ return hdfsError("read", uri_);
+ if (read == 0)
+ return Status::IOError("Unexpected EOF reading HDFS file: ",
uri_);
+ read_bytes_ += read;
+ total += read;
+ }
+ return total;
+ }
+};
+
+class HdfsPaimonFileSystem final : public ::paimon::FileSystem
+{
+ std::shared_ptr<HdfsConnection> connection_;
+
+public:
+ HdfsPaimonFileSystem(const std::string & table_path, DB::ContextPtr
context)
+ {
+ Poco::URI uri(table_path);
+ connection_ = std::make_shared<HdfsConnection>(uri.getScheme() + "://"
+ uri.getAuthority(), context);
+ }
+
+ ::paimon::Result<std::unique_ptr<::paimon::InputStream>> Open(const
std::string & path) const override
+ {
+ auto hdfs_path = pathPart(path);
+ auto * info = hdfsGetPathInfo(connection_->fs.get(),
hdfs_path.c_str());
+ if (!info)
+ return hdfsError("stat", path);
+ int64_t length = info->mSize;
+ hdfsFreeFileInfo(info, 1);
+ auto file = hdfsOpenFile(connection_->fs.get(), hdfs_path.c_str(),
O_RDONLY, 0, 0, 0);
+ if (!file)
+ return hdfsError("open", path);
+ return
std::unique_ptr<::paimon::InputStream>(std::make_unique<HdfsInputStream>(connection_,
file, path, length));
+ }
+
+ ::paimon::Result<bool> Exists(const std::string & path) const override
+ {
+ auto hdfs_path = pathPart(path);
+ // hdfsExists() in libhdfs3 returns -1 for a missing path without
+ // reliably setting errno. Stat the path instead so the missing-path
+ // error can be identified from the HDFS error reported by the API.
+ auto * info = hdfsGetPathInfo(connection_->fs.get(),
hdfs_path.c_str());
+ if (info)
+ {
+ hdfsFreeFileInfo(info, 1);
+ return true;
+ }
+ const char * error = hdfsGetLastError();
+ if (error)
+ {
+ std::string message(error);
+ if (message.find("FileNotFoundException") != std::string::npos
+ || message.find("No such file") != std::string::npos
+ || message.find("does not exist") != std::string::npos
+ || message.find("not found") != std::string::npos)
+ return false;
+ }
+ return hdfsError("exists", path);
+ }
+
+ ::paimon::Result<std::unique_ptr<::paimon::FileStatus>>
GetFileStatus(const std::string & path) const override
+ {
+ auto hdfs_path = pathPart(path);
+ auto * info = hdfsGetPathInfo(connection_->fs.get(),
hdfs_path.c_str());
+ if (!info)
+ return hdfsError("stat", path);
+ std::unique_ptr<::paimon::FileStatus> status =
std::make_unique<HdfsStatus>(*info);
+ hdfsFreeFileInfo(info, 1);
+ return status;
+ }
+
+ Status ListDir(const std::string & path,
std::vector<std::unique_ptr<::paimon::BasicFileStatus>> * result) const override
+ {
+ int count = 0;
+ auto hdfs_path = pathPart(path);
+ errno = 0;
+ auto * files = hdfsListDirectory(connection_->fs.get(),
hdfs_path.c_str(), &count);
+ if (!files && errno != 0)
+ return hdfsError("list", path);
+ for (int i = 0; i < count; ++i)
+ result->push_back(std::make_unique<HdfsStatus>(files[i]));
+ if (files)
+ hdfsFreeFileInfo(files, count);
+ return Status::OK();
+ }
+
+ Status ListFileStatus(const std::string & path,
std::vector<std::unique_ptr<::paimon::FileStatus>> * result) const override
+ {
+ int count = 0;
+ auto hdfs_path = pathPart(path);
+ errno = 0;
+ auto * files = hdfsListDirectory(connection_->fs.get(),
hdfs_path.c_str(), &count);
+ if (!files && errno != 0)
+ return hdfsError("list", path);
+ for (int i = 0; i < count; ++i)
+ result->push_back(std::make_unique<HdfsStatus>(files[i]));
+ if (files)
+ hdfsFreeFileInfo(files, count);
+ return Status::OK();
+ }
+
+ ::paimon::Result<std::unique_ptr<::paimon::OutputStream>> Create(const
std::string &, bool) const override
+ { return Status::NotImplemented("HDFS Paimon writer is not supported"); }
+ Status Mkdirs(const std::string &) const override { return
Status::NotImplemented("HDFS Paimon writer is not supported"); }
+ Status Rename(const std::string &, const std::string &) const override {
return Status::NotImplemented("HDFS Paimon writer is not supported"); }
+ Status Delete(const std::string &, bool) const override { return
Status::NotImplemented("HDFS Paimon writer is not supported"); }
+};
+}
+
+std::shared_ptr<::paimon::FileSystem> createHdfsPaimonFileSystem(const
std::string & table_path, DB::ContextPtr context)
+{
+ return std::make_shared<HdfsPaimonFileSystem>(table_path,
std::move(context));
+}
+}
diff --git
a/gluten-paimon/src-paimon/main/java/org/apache/gluten/substrait/rel/PaimonLocalFilesBuilder.java
b/cpp-ch/local-engine/Storages/SubstraitSource/Paimon/HdfsPaimonFileSystem.h
similarity index 55%
copy from
gluten-paimon/src-paimon/main/java/org/apache/gluten/substrait/rel/PaimonLocalFilesBuilder.java
copy to
cpp-ch/local-engine/Storages/SubstraitSource/Paimon/HdfsPaimonFileSystem.h
index 4d6c33baee..2ab35bdc10 100644
---
a/gluten-paimon/src-paimon/main/java/org/apache/gluten/substrait/rel/PaimonLocalFilesBuilder.java
+++ b/cpp-ch/local-engine/Storages/SubstraitSource/Paimon/HdfsPaimonFileSystem.h
@@ -14,29 +14,14 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.gluten.substrait.rel;
+#pragma once
-import java.util.List;
-import java.util.Map;
+#include <paimon/fs/file_system.h>
-public class PaimonLocalFilesBuilder {
- public static PaimonLocalFilesNode makePaimonLocalFiles(
- Integer index,
- List<String> paths,
- List<Long> starts,
- List<Long> lengths,
- List<Map<String, String>> partitionColumns,
- LocalFilesNode.ReadFileFormat fileFormat,
- List<String> preferredLocations,
- Map<String, String> properties) {
- return new PaimonLocalFilesNode(
- index,
- paths,
- starts,
- lengths,
- partitionColumns,
- fileFormat,
- preferredLocations,
- properties);
- }
+namespace DB { class Context; using ContextPtr = std::shared_ptr<const
Context>; }
+
+namespace local_engine::paimon
+{
+/// Read-only HDFS adapter for Paimon's native split reader.
+std::shared_ptr<::paimon::FileSystem> createHdfsPaimonFileSystem(const
std::string & table_path, DB::ContextPtr context);
}
diff --git
a/cpp-ch/local-engine/Storages/SubstraitSource/Paimon/PaimonFormatFile.cpp
b/cpp-ch/local-engine/Storages/SubstraitSource/Paimon/PaimonFormatFile.cpp
new file mode 100644
index 0000000000..404d0c2f57
--- /dev/null
+++ b/cpp-ch/local-engine/Storages/SubstraitSource/Paimon/PaimonFormatFile.cpp
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "PaimonFormatFile.h"
+#include "PaimonReader.h"
+
+#include <Common/Exception.h>
+
+namespace DB::ErrorCodes
+{
+extern const int BAD_ARGUMENTS;
+}
+
+namespace local_engine
+{
+
+PaimonFormatFile::PaimonFormatFile(
+ DB::ContextPtr context, const substrait::ReadRel::LocalFiles::FileOrFiles
& file, const ReadBufferBuilderPtr & read_buffer_builder)
+ : FormatFile(context, file, read_buffer_builder)
+{
+ if (!file.has_paimon() || !file.paimon().has_split_read_spec() ||
file.paimon().split_read_spec().table_path().empty()
+ || file.paimon().split_read_spec().serialized_data_split().empty())
+ throw DB::Exception(
+ DB::ErrorCodes::BAD_ARGUMENTS,
+ "Paimon native read requires table_path and serialized_data_split
in substrait LocalFiles");
+}
+
+FormatFile::InputFormatPtr PaimonFormatFile::createInputFormat(
+ const DB::Block & header, const std::shared_ptr<const DB::ActionsDAG> &
/*filter_actions_dag*/)
+{
+ return paimon::createPaimonInputFormat(*this, header);
+}
+
+}
diff --git
a/gluten-paimon/src-paimon/main/java/org/apache/gluten/substrait/rel/PaimonLocalFilesBuilder.java
b/cpp-ch/local-engine/Storages/SubstraitSource/Paimon/PaimonFormatFile.h
similarity index 54%
copy from
gluten-paimon/src-paimon/main/java/org/apache/gluten/substrait/rel/PaimonLocalFilesBuilder.java
copy to cpp-ch/local-engine/Storages/SubstraitSource/Paimon/PaimonFormatFile.h
index 4d6c33baee..d02e45fb06 100644
---
a/gluten-paimon/src-paimon/main/java/org/apache/gluten/substrait/rel/PaimonLocalFilesBuilder.java
+++ b/cpp-ch/local-engine/Storages/SubstraitSource/Paimon/PaimonFormatFile.h
@@ -14,29 +14,27 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.gluten.substrait.rel;
+#pragma once
-import java.util.List;
-import java.util.Map;
+#include <Storages/SubstraitSource/FormatFile.h>
+
+namespace local_engine
+{
+/// Carries Substrait Paimon read options and creates the native input format.
+class PaimonFormatFile final : public FormatFile
+{
+public:
+ PaimonFormatFile(
+ DB::ContextPtr context,
+ const substrait::ReadRel::LocalFiles::FileOrFiles & file,
+ const ReadBufferBuilderPtr & read_buffer_builder);
+
+ InputFormatPtr
+ createInputFormat(const DB::Block & header, const std::shared_ptr<const
DB::ActionsDAG> & filter_actions_dag = nullptr) override;
+
+ String getFileFormat() const override { return "Paimon"; }
+
+ bool supportSplit() const override { return false; }
+};
-public class PaimonLocalFilesBuilder {
- public static PaimonLocalFilesNode makePaimonLocalFiles(
- Integer index,
- List<String> paths,
- List<Long> starts,
- List<Long> lengths,
- List<Map<String, String>> partitionColumns,
- LocalFilesNode.ReadFileFormat fileFormat,
- List<String> preferredLocations,
- Map<String, String> properties) {
- return new PaimonLocalFilesNode(
- index,
- paths,
- starts,
- lengths,
- partitionColumns,
- fileFormat,
- preferredLocations,
- properties);
- }
}
diff --git
a/cpp-ch/local-engine/Storages/SubstraitSource/Paimon/PaimonReader.cpp
b/cpp-ch/local-engine/Storages/SubstraitSource/Paimon/PaimonReader.cpp
new file mode 100644
index 0000000000..85e58eb941
--- /dev/null
+++ b/cpp-ch/local-engine/Storages/SubstraitSource/Paimon/PaimonReader.cpp
@@ -0,0 +1,282 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "PaimonReader.h"
+#include "HdfsPaimonFileSystem.h"
+
+#include <Formats/FormatFactory.h>
+#include <Formats/FormatSettings.h>
+#include <IO/ReadBufferFromString.h>
+#include <Processors/Formats/Impl/ArrowColumnToCHColumn.h>
+#include <Common/BlockTypeUtils.h>
+#include <Common/CHUtil.h>
+#include <Common/Exception.h>
+#include <Common/logger_useful.h>
+#include <arrow/api.h>
+#include <arrow/c/bridge.h>
+#include <paimon/memory/memory_pool.h>
+#include <paimon/read_context.h>
+#include <paimon/reader/batch_reader.h>
+#include <paimon/reader/count_reader.h>
+#include <paimon/table/source/split.h>
+#include <paimon/table/source/table_read.h>
+
+#include <algorithm>
+#include <map>
+#include <utility>
+#include <Poco/URI.h>
+
+namespace DB::ErrorCodes
+{
+extern const int BAD_ARGUMENTS;
+extern const int LOGICAL_ERROR;
+}
+
+namespace local_engine::paimon
+{
+namespace
+{
+template <typename T>
+T takeResult(::paimon::Result<T> && result)
+{
+ if (!result.ok())
+ throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, "Paimon reader:
{}", result.status().ToString());
+ return std::move(result).value();
+}
+
+class PaimonCountInputFormat final : public DB::IInputFormat
+{
+ std::unique_ptr<::paimon::CountReader> reader_;
+ String table_path_;
+ int64_t remaining_rows_ = -1;
+
+public:
+ PaimonCountInputFormat(
+ DB::ReadBuffer * in_,
+ const DB::SharedHeader & header_,
+ std::unique_ptr<::paimon::CountReader> reader,
+ String table_path)
+ : DB::IInputFormat(header_, in_)
+ , reader_(std::move(reader))
+ , table_path_(std::move(table_path))
+ {
+ }
+
+ String getName() const override { return "PaimonCountInputFormat"; }
+
+protected:
+ DB::Chunk read() override
+ {
+ if (remaining_rows_ < 0)
+ {
+ remaining_rows_ = takeResult(reader_->CountRows());
+ if (remaining_rows_ < 0)
+ throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "Paimon
CountReader returned a negative row count");
+ LOG_DEBUG(&Poco::Logger::get("PaimonReader"), "Native Paimon count
finished: table_path={}, rows={}", table_path_, remaining_rows_);
+ reader_.reset();
+ }
+ if (remaining_rows_ == 0)
+ return {};
+
+ constexpr int64_t count_batch_size = 8192;
+ const auto rows = std::min(remaining_rows_, count_batch_size);
+ remaining_rows_ -= rows;
+ return BlockUtil::buildRowCountChunk(rows);
+ }
+};
+
+class PaimonBlockInputFormat final : public DB::IInputFormat
+{
+ std::unique_ptr<::paimon::BatchReader> reader_;
+ String table_path_;
+ DB::ArrowColumnToCHColumn arrow_to_ch_;
+ bool eof_ = false;
+ size_t batches_read_ = 0;
+ size_t rows_read_ = 0;
+
+public:
+ PaimonBlockInputFormat(
+ DB::ReadBuffer * in_,
+ const DB::SharedHeader & header_,
+ std::unique_ptr<::paimon::BatchReader> reader,
+ String table_path,
+ DB::ContextPtr context)
+ : DB::IInputFormat(header_, in_)
+ , reader_(std::move(reader))
+ , table_path_(std::move(table_path))
+ , arrow_to_ch_(
+ [&]
+ {
+ const DB::FormatSettings format_settings =
getFormatSettings(context);
+ return DB::ArrowColumnToCHColumn(
+ *header_,
+ "Paimon",
+ format_settings,
+ std::nullopt,
+ std::nullopt,
+ format_settings.parquet.allow_missing_columns,
+ format_settings.null_as_default,
+ format_settings.date_time_overflow_behavior,
+ format_settings.parquet.allow_geoparquet_parser,
+ format_settings.parquet.case_insensitive_column_matching,
+ false,
+ format_settings.parquet.enable_json_parsing);
+ }())
+ {
+ }
+
+ String getName() const override { return "PaimonBlockInputFormat"; }
+
+protected:
+ DB::Chunk read() override
+ {
+ if (eof_ || !reader_)
+ return {};
+
+ while (true)
+ {
+ auto paimon_batch = takeResult(reader_->NextBatch());
+ if (::paimon::BatchReader::IsEofBatch(paimon_batch))
+ {
+ LOG_INFO(
+ &Poco::Logger::get("PaimonReader"),
+ "Native Paimon read finished: table_path={}, batches={},
rows={}",
+ table_path_,
+ batches_read_,
+ rows_read_);
+ eof_ = true;
+ reader_->Close();
+ reader_.reset();
+ return {};
+ }
+
+ auto import_result = arrow::ImportArray(paimon_batch.first.get(),
paimon_batch.second.get());
+ if (!import_result.ok())
+ {
+ if (paimon_batch.first && paimon_batch.first->release)
+ paimon_batch.first->release(paimon_batch.first.get());
+ if (paimon_batch.second && paimon_batch.second->release)
+ paimon_batch.second->release(paimon_batch.second.get());
+ throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, "Paimon
Arrow import: {}", import_result.status().ToString());
+ }
+
+ const auto struct_array =
std::dynamic_pointer_cast<arrow::StructArray>(import_result.ValueOrDie());
+ if (!struct_array)
+ throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, "Paimon
reader expects a struct Arrow batch");
+
+ const auto struct_type =
std::static_pointer_cast<arrow::StructType>(struct_array->type());
+ std::vector<std::shared_ptr<arrow::Field>> fields;
+ std::vector<std::shared_ptr<arrow::Array>> columns;
+ for (int i = 0; i < struct_type->num_fields(); ++i)
+ {
+ if (struct_type->field(i)->name() == "_VALUE_KIND")
+ continue;
+ fields.push_back(struct_type->field(i));
+ columns.push_back(struct_array->field(i));
+ }
+ if (fields.empty())
+ throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, "Paimon
batch has no data columns");
+
+ auto batch = arrow::RecordBatch::Make(arrow::schema(fields),
struct_array->length(), columns);
+ auto table_result = arrow::Table::FromRecordBatches({batch});
+ if (!table_result.ok())
+ throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, "Paimon
Arrow table: {}", table_result.status().ToString());
+ auto table = table_result.ValueOrDie();
+ DB::Chunk chunk = arrow_to_ch_.arrowTableToCHChunk(table,
table->num_rows(), nullptr, nullptr);
+ if (chunk.getNumRows() != 0)
+ {
+ ++batches_read_;
+ rows_read_ += chunk.getNumRows();
+ if (batches_read_ == 1 || batches_read_ % 1000 == 0)
+ LOG_DEBUG(
+ &Poco::Logger::get("PaimonReader"),
+ "Native Paimon reader emitted batch: table_path={},
batch={}, rows={}",
+ table_path_,
+ batches_read_,
+ chunk.getNumRows());
+ return chunk;
+ }
+ }
+ }
+};
+
+}
+
+FormatFile::InputFormatPtr createPaimonInputFormat(const FormatFile & file,
const DB::Block & header)
+{
+ const auto & file_info = file.getFileInfo();
+ if (!file_info.has_paimon())
+ throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "Paimon input
format requires paimon read options in LocalFiles");
+
+ const auto & split_read_spec = file_info.paimon().split_read_spec();
+ const auto & serialized_split = split_read_spec.serialized_data_split();
+ const auto & table_path = split_read_spec.table_path();
+
+ LOG_INFO(
+ &Poco::Logger::get("PaimonReader"),
+ "Opening native Paimon reader: table_path={},
serialized_split_bytes={}",
+ table_path,
+ serialized_split.size());
+
+ auto split = takeResult(::paimon::Split::Deserialize(
+ serialized_split.data(), serialized_split.size(),
::paimon::GetDefaultPool()));
+ ::paimon::ReadContextBuilder builder(table_path);
+ const std::map<std::string, std::string> read_options(
+ split_read_spec.read_options().begin(),
split_read_spec.read_options().end());
+ builder.SetOptions(read_options);
+ builder.WithBranch(split_read_spec.branch());
+ const bool count_only = header.columns() == 1 &&
header.has(BlockUtil::VIRTUAL_ROW_COUNT_COLUMN);
+ std::vector<std::string> read_fields;
+ read_fields.reserve(header.columns());
+ for (const auto & column : header)
+ if (column.name != BlockUtil::VIRTUAL_ROW_COUNT_COLUMN)
+ read_fields.push_back(column.name);
+ if (!read_fields.empty())
+ builder.SetReadFieldNames(read_fields);
+ const auto & config = file.getContext()->getConfigRef();
+ if (config.has("paimon.read_batch_size"))
+ {
+ const int batch_size = config.getInt("paimon.read_batch_size");
+ if (batch_size <= 0)
+ throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS,
"paimon.read_batch_size must be positive, got {}", batch_size);
+ builder.AddOption("read.batch-size", std::to_string(batch_size));
+ }
+ if (Poco::URI(table_path).getScheme() == "hdfs")
+ builder.WithFileSystem(createHdfsPaimonFileSystem(table_path,
file.getContext()));
+ auto read_context = takeResult(builder.Finish());
+ auto table_read =
takeResult(::paimon::TableRead::Create(std::move(read_context)));
+
+ static const std::string empty_read_buffer;
+ auto read_buffer =
std::make_unique<DB::ReadBufferFromString>(empty_read_buffer);
+ DB::InputFormatPtr input;
+ if (count_only)
+ {
+ auto count_reader = takeResult(table_read->CreateCountReader({split}));
+ input = std::make_shared<PaimonCountInputFormat>(read_buffer.get(),
toShared(header), std::move(count_reader), table_path);
+ LOG_DEBUG(&Poco::Logger::get("PaimonReader"), "Native Paimon count
reader created: table_path={}", table_path);
+ }
+ else
+ {
+ auto reader = takeResult(table_read->CreateReader(split));
+ input = std::make_shared<PaimonBlockInputFormat>(
+ read_buffer.get(), toShared(header), std::move(reader),
table_path, file.getContext());
+ LOG_INFO(&Poco::Logger::get("PaimonReader"), "Native Paimon reader
created: table_path={}", table_path);
+ }
+ return std::make_shared<FormatFile::InputFormat>(std::move(read_buffer),
input);
+}
+
+}
diff --git
a/gluten-paimon/src-paimon/main/java/org/apache/gluten/substrait/rel/PaimonLocalFilesBuilder.java
b/cpp-ch/local-engine/Storages/SubstraitSource/Paimon/PaimonReader.h
similarity index 55%
copy from
gluten-paimon/src-paimon/main/java/org/apache/gluten/substrait/rel/PaimonLocalFilesBuilder.java
copy to cpp-ch/local-engine/Storages/SubstraitSource/Paimon/PaimonReader.h
index 4d6c33baee..e38d967603 100644
---
a/gluten-paimon/src-paimon/main/java/org/apache/gluten/substrait/rel/PaimonLocalFilesBuilder.java
+++ b/cpp-ch/local-engine/Storages/SubstraitSource/Paimon/PaimonReader.h
@@ -14,29 +14,12 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.gluten.substrait.rel;
+#pragma once
-import java.util.List;
-import java.util.Map;
+#include <Storages/SubstraitSource/FormatFile.h>
+
+namespace local_engine::paimon
+{
+FormatFile::InputFormatPtr createPaimonInputFormat(const FormatFile & file,
const DB::Block & header);
-public class PaimonLocalFilesBuilder {
- public static PaimonLocalFilesNode makePaimonLocalFiles(
- Integer index,
- List<String> paths,
- List<Long> starts,
- List<Long> lengths,
- List<Map<String, String>> partitionColumns,
- LocalFilesNode.ReadFileFormat fileFormat,
- List<String> preferredLocations,
- Map<String, String> properties) {
- return new PaimonLocalFilesNode(
- index,
- paths,
- starts,
- lengths,
- partitionColumns,
- fileFormat,
- preferredLocations,
- properties);
- }
}
diff --git a/docs/backends/clickhouse/get-started/ClickHouse.md
b/docs/backends/clickhouse/get-started/ClickHouse.md
index ea2adff8cf..59522dbdbd 100644
--- a/docs/backends/clickhouse/get-started/ClickHouse.md
+++ b/docs/backends/clickhouse/get-started/ClickHouse.md
@@ -183,6 +183,61 @@ The prerequisites are the same as the one mentioned above.
Compile Gluten with C
ls -al
backends-clickhouse/target/gluten-XXXXX-spark-3.5-jar-with-dependencies.jar
```
+#### Read Paimon tables
+
+Build with the `paimon` profile in addition to the ClickHouse and Delta
profiles:
+
+```shell
+./build/mvn install -Pbackends-clickhouse,delta,paimon,spark-3.5 -DskipTests
+```
+
+Put the matching `paimon-spark` runtime JAR on the Spark classpath and
configure
+the Paimon Spark session extension and a catalog, for example:
+
+```text
+spark.sql.extensions=org.apache.paimon.spark.extensions.PaimonSparkSessionExtensions
+spark.sql.catalog.paimon=org.apache.paimon.spark.SparkCatalog
+spark.sql.catalog.paimon.warehouse=file:///path/to/paimon-warehouse
+```
+
+Paimon plans the file splits in Spark. Gluten passes serialized splits to the
+ClickHouse native Paimon reader, including for primary-key tables.
+
+A native reader uses the Paimon C++ API from `libch.so` and
+links it against a separate shared
+[`paimon-cpp`](https://github.com/apache/paimon-cpp) v0.3.0 build. The build
+host needs libc++ development headers and libraries for the selected Clang
+version. The libc++, libc++abi, and libunwind runtimes are linked statically
+into `libpaimon.so`. With `-DGLUTEN_ENABLE_NATIVE_PAIMON=ON` (the default),
+the wrapper CMake first builds ClickHouse's libc++ runtime, then builds
+paimon-cpp with its file-system and format components in one `libpaimon.so`,
+copies that library next to `libch.so`, and sets
+`RPATH` to `$ORIGIN` for loading. Rebuild the native backend before building
+the JVM modules. Gluten passes serialized Paimon splits to the C++ reader by
+default. The reader requests only the columns required by the
+query. For HDFS tables, configure
+`spark.gluten.sql.columnar.backend.ch.runtime_config.hdfs.libhdfs3_conf`
+with the HDFS site XML on every executor. The optional
+`spark.gluten.sql.columnar.backend.ch.runtime_config.paimon.read_batch_size=8192`
+sets the Paimon C++ reader batch size; when omitted, the SDK default applies.
+Primary-key tables use the native Paimon reader by default.
+Ship `libch.so` and `libpaimon.so` on every executor.
+The bundled `libpaimon.so` binds the math functions used by Paimon to their
+original glibc versions and provides a fallback for `pthread_cond_clockwait`.
+The current build has a glibc 2.17 baseline; check the actual result after
+each build. Deploy both libraries from the same build; older `libpaimon.so`
+files can still require glibc 2.30.
+Build these libraries on a system whose glibc is no newer than that on the
+oldest driver or executor node. A build made on a newer system can fail before
+Spark starts with `GLIBC_*` not found errors. Rebuild libch and bundled
paimon-cpp dependencies in a fresh
+build workspace on a compatible system, then deploy the newly built libraries
+together. Changing the Paimon read configuration does not resolve a binary
+compatibility error. Check the requirements before deployment with
+`readelf --version-info libch.so` and `readelf --version-info libpaimon.so`,
+then compare against the target system with `ldd --version`.
+Set `GLUTEN_TEST_NATIVE_PAIMON=1` when running `ClickHousePaimonSuite` with a
+native-enabled build to exercise the native read path.
+
### Gluten in local Spark Thrift Server
#### Prepare working directory
@@ -699,5 +754,3 @@ spark.celeborn.storage.hdfs.dir hdfs://<namenode>/celeborn
# please refer to this URL
(https://github.com/apache/celeborn/tree/main/assets/spark-patch) to apply the
patch into your own Spark.
spark.dynamicAllocation.enabled false
```
-
-
diff --git a/ep/build-clickhouse/src/build-clickhouse.sh
b/ep/build-clickhouse/src/build-clickhouse.sh
index 7441e714ef..7b8178aa8b 100644
--- a/ep/build-clickhouse/src/build-clickhouse.sh
+++ b/ep/build-clickhouse/src/build-clickhouse.sh
@@ -17,6 +17,7 @@
export GLUTEN_SOURCE=$(builtin cd $(dirname $0)/../../..; pwd)
export CH_SOURCE_DIR=${GLUTEN_SOURCE}/cpp-ch/ClickHouse
CMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE:-RelWithDebInfo}
+GLUTEN_ENABLE_NATIVE_PAIMON=${GLUTEN_ENABLE_NATIVE_PAIMON:-ON}
for arg in "$@"
do
@@ -40,5 +41,8 @@ echo ${GLUTEN_SOURCE}
export CC=${CC:-clang-18}
export CXX=${CXX:-clang++-18}
-cmake -G Ninja -S ${GLUTEN_SOURCE}/cpp-ch -B ${GLUTEN_SOURCE}/cpp-ch/build_ch
-DCH_SOURCE_DIR=${CH_SOURCE_DIR} "-DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}"
+cmake -G Ninja -S ${GLUTEN_SOURCE}/cpp-ch -B ${GLUTEN_SOURCE}/cpp-ch/build_ch \
+ -DCH_SOURCE_DIR=${CH_SOURCE_DIR} \
+ "-DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}" \
+ "-DGLUTEN_ENABLE_NATIVE_PAIMON=${GLUTEN_ENABLE_NATIVE_PAIMON}"
cmake --build ${GLUTEN_SOURCE}/cpp-ch/build_ch --target build_ch
diff --git a/ep/build-clickhouse/src/install-ubuntu.sh
b/ep/build-clickhouse/src/install-ubuntu.sh
index 4ef11f919e..8a29721156 100644
--- a/ep/build-clickhouse/src/install-ubuntu.sh
+++ b/ep/build-clickhouse/src/install-ubuntu.sh
@@ -17,7 +17,7 @@
# ref: https://clickhouse.com/docs/en/development/build
echo "Install Prerequisites"
-sudo apt-get install git cmake ccache python3 ninja-build nasm yasm gawk
lsb-release wget software-properties-common gnupg
+sudo apt-get install git cmake ccache patch python3 ninja-build nasm yasm gawk
lsb-release wget software-properties-common gnupg
echo "Install and Use the Clang compiler"
sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)"
diff --git a/ep/build-clickhouse/src/package.sh
b/ep/build-clickhouse/src/package.sh
index a796a1074e..3086185387 100755
--- a/ep/build-clickhouse/src/package.sh
+++ b/ep/build-clickhouse/src/package.sh
@@ -16,6 +16,13 @@
set -exu
+# Set to ON to include both native and JVM Paimon support.
+export GLUTEN_ENABLE_NATIVE_PAIMON=${GLUTEN_ENABLE_NATIVE_PAIMON:-ON}
+case "${GLUTEN_ENABLE_NATIVE_PAIMON}" in
+ ON|OFF) ;;
+ *) echo "GLUTEN_ENABLE_NATIVE_PAIMON must be ON or OFF" >&2; exit 1 ;;
+esac
+
GLUTEN_SOURCE=$(cd $(dirname -- $0)/../../..; pwd -P)
function detect_os_version() {
@@ -36,8 +43,7 @@ detect_os_version
DEFAULT_SPARK_PROFILE="spark-3.5"
function get_project_version() {
cd "${GLUTEN_SOURCE}"
- # use mvn command to get project version
- PROJECT_VERSION=$(mvn -q -P${DEFAULT_SPARK_PROFILE} -Dexec.executable="echo"
-Dexec.args='${project.version}' --non-recursive exec:exec)
+ PROJECT_VERSION=$("${GLUTEN_SOURCE}/build/mvn" -q -P${DEFAULT_SPARK_PROFILE}
-Dexec.executable="echo" -Dexec.args='${project.version}' --non-recursive
exec:exec)
export PROJECT_VERSION=${PROJECT_VERSION}
}
get_project_version
@@ -90,16 +96,28 @@ function build_gluten_by_spark_version() {
sv=$(echo "$spark_profile" | tr -d '.')
echo "build gluten with spark ${spark_profile}, scala ${scala_version}"
- mvn clean install -Pbackends-clickhouse -Pspark-"${spark_profile}"
-Pscala-"${scala_version}" -Pceleborn -Piceberg -Pdelta -DskipTests
-Dcheckstyle.skip
+ local paimon_profiles=()
+ if [[ "${GLUTEN_ENABLE_NATIVE_PAIMON}" == "ON" ]]; then
+ paimon_profiles+=(-Ppaimon)
+ fi
+ "${GLUTEN_SOURCE}/build/mvn" clean install -Pbackends-clickhouse
-Pspark-"${spark_profile}" -Pscala-"${scala_version}" -Pceleborn -Piceberg
-Pdelta "${paimon_profiles[@]}" -DskipTests -Dcheckstyle.skip
cp
"${GLUTEN_SOURCE}"/backends-clickhouse/target/gluten-*-spark-"${spark_profile}"-jar-with-dependencies.jar
"${PACKAGE_DIR_PATH}"/jars/spark"${sv}"/gluten.jar
- delta_version=$(mvn -q -Dexec.executable="echo"
-Dexec.args='${delta.version}' -Pspark-"${spark_profile}" --non-recursive
exec:exec)
- delta_package_name=$(mvn -q -Dexec.executable="echo"
-Dexec.args='${delta.package.name}' -Pspark-"${spark_profile}" --non-recursive
exec:exec)
+ delta_version=$("${GLUTEN_SOURCE}/build/mvn" -q -Dexec.executable="echo"
-Dexec.args='${delta.version}' -Pspark-"${spark_profile}" --non-recursive
exec:exec)
+ delta_package_name=$("${GLUTEN_SOURCE}/build/mvn" -q
-Dexec.executable="echo" -Dexec.args='${delta.package.name}'
-Pspark-"${spark_profile}" --non-recursive exec:exec)
wget
https://repo1.maven.org/maven2/io/delta/"${delta_package_name}"_${scala_version}/"${delta_version}"/"${delta_package_name}"_${scala_version}-"${delta_version}".jar
-P "${PACKAGE_DIR_PATH}"/jars/spark"${sv}"
wget
https://repo1.maven.org/maven2/io/delta/delta-storage/"${delta_version}"/delta-storage-"${delta_version}".jar
-P "${PACKAGE_DIR_PATH}"/jars/spark"${sv}"
- celeborn_version=$(mvn -q -P${DEFAULT_SPARK_PROFILE}
-Dexec.executable="echo" -Dexec.args='${celeborn.version}' --non-recursive
exec:exec)
+ celeborn_version=$("${GLUTEN_SOURCE}/build/mvn" -q
-P${DEFAULT_SPARK_PROFILE} -Dexec.executable="echo"
-Dexec.args='${celeborn.version}' --non-recursive exec:exec)
wget
https://repo1.maven.org/maven2/org/apache/celeborn/celeborn-client-spark-3-shaded_${scala_version}/${celeborn_version}/celeborn-client-spark-3-shaded_${scala_version}-${celeborn_version}.jar
-P "${PACKAGE_DIR_PATH}"/jars/spark"${sv}"
}
+# build libch.so
+bash "${GLUTEN_SOURCE}"/ep/build-clickhouse/src/build-clickhouse.sh
+LIBCH_DIR="$GLUTEN_SOURCE/cpp-ch/build/utils/extern-local-engine"
+cp "${LIBCH_DIR}"/libch.so "${PACKAGE_DIR_PATH}"/libs/libch.so
+if [[ "${GLUTEN_ENABLE_NATIVE_PAIMON}" == "ON" ]]; then
+ cp "${LIBCH_DIR}/libpaimon.so" "${PACKAGE_DIR_PATH}"/libs/
+fi
+
for ssv in "${spark_scala_versions[@]}"
do
spark_profile="${ssv%_*}"
@@ -107,10 +125,6 @@ do
build_gluten_by_spark_version "$spark_profile" "$scala_version"
done
-# build libch.so
-bash "${GLUTEN_SOURCE}"/ep/build-clickhouse/src/build-clickhouse.sh
-cp "$GLUTEN_SOURCE"/cpp-ch/build/utils/extern-local-engine/libch.so
"${PACKAGE_DIR_PATH}"/libs/libch.so
-
# copy bin and conf
cp "${GLUTEN_SOURCE}"/ep/build-clickhouse/src/resources/bin/*
"${GLUTEN_SOURCE}"/dist/"${PACKAGE_NAME}"/bin
cp "${GLUTEN_SOURCE}"/ep/build-clickhouse/src/resources/conf/*
"${GLUTEN_SOURCE}"/dist/"${PACKAGE_NAME}"/conf
diff --git
a/gluten-paimon/src-paimon/main/java/org/apache/gluten/substrait/rel/PaimonLocalFilesBuilder.java
b/gluten-paimon/src-paimon/main/java/org/apache/gluten/substrait/rel/PaimonLocalFilesBuilder.java
index 4d6c33baee..f671a74afc 100644
---
a/gluten-paimon/src-paimon/main/java/org/apache/gluten/substrait/rel/PaimonLocalFilesBuilder.java
+++
b/gluten-paimon/src-paimon/main/java/org/apache/gluten/substrait/rel/PaimonLocalFilesBuilder.java
@@ -20,6 +20,17 @@ import java.util.List;
import java.util.Map;
public class PaimonLocalFilesBuilder {
+ public static PaimonLocalFilesNode makeNativePaimonLocalFiles(
+ Integer index,
+ String tablePath,
+ String branch,
+ Map<String, String> readOptions,
+ List<byte[]> splits,
+ List<String> preferredLocations) {
+ return new PaimonLocalFilesNode(
+ index, tablePath, branch, readOptions, splits, preferredLocations);
+ }
+
public static PaimonLocalFilesNode makePaimonLocalFiles(
Integer index,
List<String> paths,
diff --git
a/gluten-paimon/src-paimon/main/java/org/apache/gluten/substrait/rel/PaimonLocalFilesNode.java
b/gluten-paimon/src-paimon/main/java/org/apache/gluten/substrait/rel/PaimonLocalFilesNode.java
index 30f47d87b3..5b9584b242 100644
---
a/gluten-paimon/src-paimon/main/java/org/apache/gluten/substrait/rel/PaimonLocalFilesNode.java
+++
b/gluten-paimon/src-paimon/main/java/org/apache/gluten/substrait/rel/PaimonLocalFilesNode.java
@@ -16,11 +16,18 @@
*/
package org.apache.gluten.substrait.rel;
+import com.google.protobuf.ByteString;
+import io.substrait.proto.ReadRel;
+
import java.util.ArrayList;
+import java.util.Collections;
import java.util.List;
import java.util.Map;
public class PaimonLocalFilesNode extends LocalFilesNode {
+ private final List<byte[]> serializedSplits;
+ private final String branch;
+ private final Map<String, String> readOptions;
public PaimonLocalFilesNode(
Integer index,
@@ -44,5 +51,50 @@ public class PaimonLocalFilesNode extends LocalFilesNode {
preferredLocations,
properties,
new ArrayList<>());
+ this.serializedSplits = null;
+ this.branch = null;
+ this.readOptions = Collections.emptyMap();
+ }
+
+ public PaimonLocalFilesNode(
+ Integer index,
+ String tablePath,
+ String branch,
+ Map<String, String> readOptions,
+ List<byte[]> splits,
+ List<String> preferredLocations) {
+ super(
+ index,
+ Collections.nCopies(splits.size(), tablePath),
+ Collections.nCopies(splits.size(), 0L),
+ Collections.nCopies(splits.size(), 1L),
+ new ArrayList<>(),
+ new ArrayList<>(),
+ Collections.nCopies(splits.size(), Collections.emptyMap()),
+ new ArrayList<>(),
+ ReadFileFormat.UnknownFormat,
+ preferredLocations,
+ Collections.emptyMap(),
+ new ArrayList<>());
+ this.serializedSplits = splits;
+ this.branch = branch;
+ this.readOptions = readOptions;
+ }
+
+ @Override
+ protected void processFileBuilder(ReadRel.LocalFiles.FileOrFiles.Builder
fileBuilder, int index) {
+ if (serializedSplits != null) {
+ ReadRel.LocalFiles.FileOrFiles.PaimonReadOptions.PaimonSplitReadSpec
splitReadSpec =
+
ReadRel.LocalFiles.FileOrFiles.PaimonReadOptions.PaimonSplitReadSpec.newBuilder()
+ .setTablePath(getPaths().get(index))
+
.setSerializedDataSplit(ByteString.copyFrom(serializedSplits.get(index)))
+ .setBranch(branch)
+ .putAllReadOptions(readOptions)
+ .build();
+ fileBuilder.setPaimon(
+ ReadRel.LocalFiles.FileOrFiles.PaimonReadOptions.newBuilder()
+ .setSplitReadSpec(splitReadSpec)
+ .build());
+ }
}
}
diff --git
a/gluten-substrait/src/main/resources/substrait/proto/substrait/algebra.proto
b/gluten-substrait/src/main/resources/substrait/proto/substrait/algebra.proto
index 562593ba91..fc8ece5865 100644
---
a/gluten-substrait/src/main/resources/substrait/proto/substrait/algebra.proto
+++
b/gluten-substrait/src/main/resources/substrait/proto/substrait/algebra.proto
@@ -251,6 +251,12 @@ message ReadRel {
}
message PaimonReadOptions {
+ message PaimonSplitReadSpec {
+ bytes serialized_data_split = 1;
+ string table_path = 2;
+ string branch = 3;
+ map<string, string> read_options = 4;
+ }
int32 bucket = 1;
int64 first_row_id = 2;
int64 max_sequence_number = 3;
@@ -262,6 +268,8 @@ message ReadRel {
bool use_hive_split = 7;
bool raw_convertible = 8;
repeated string primary_keys = 9;
+ // Native Paimon reader input.
+ optional PaimonSplitReadSpec split_read_spec = 10;
}
// File reading options
diff --git a/gluten-ut/spark35/pom.xml b/gluten-ut/spark35/pom.xml
index 22b99d4fa0..2bb8e7db0c 100644
--- a/gluten-ut/spark35/pom.xml
+++ b/gluten-ut/spark35/pom.xml
@@ -111,6 +111,17 @@
</build>
<profiles>
+ <profile>
+ <id>paimon</id>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.gluten</groupId>
+ <artifactId>gluten-paimon</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+ </profile>
<profile>
<id>backends-clickhouse</id>
<activation>
diff --git
a/gluten-paimon/src-paimon/main/java/org/apache/gluten/substrait/rel/PaimonLocalFilesNode.java
b/gluten-ut/spark35/src-paimon/test/scala/org/apache/gluten/substrait/rel/PaimonLocalFilesSuite.scala
similarity index 50%
copy from
gluten-paimon/src-paimon/main/java/org/apache/gluten/substrait/rel/PaimonLocalFilesNode.java
copy to
gluten-ut/spark35/src-paimon/test/scala/org/apache/gluten/substrait/rel/PaimonLocalFilesSuite.scala
index 30f47d87b3..d4ae3825ab 100644
---
a/gluten-paimon/src-paimon/main/java/org/apache/gluten/substrait/rel/PaimonLocalFilesNode.java
+++
b/gluten-ut/spark35/src-paimon/test/scala/org/apache/gluten/substrait/rel/PaimonLocalFilesSuite.scala
@@ -14,35 +14,25 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.gluten.substrait.rel;
+package org.apache.gluten.substrait.rel
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
+import org.scalatest.funsuite.AnyFunSuite
-public class PaimonLocalFilesNode extends LocalFilesNode {
+import scala.collection.JavaConverters._
- public PaimonLocalFilesNode(
- Integer index,
- List<String> paths,
- List<Long> starts,
- List<Long> lengths,
- List<Map<String, String>> partitionColumns,
- ReadFileFormat fileFormat,
- List<String> preferredLocations,
- Map<String, String> properties) {
- super(
- index,
- paths,
- starts,
- lengths,
- new ArrayList<>(),
- new ArrayList<>(),
- partitionColumns,
- new ArrayList<>(),
- fileFormat,
- preferredLocations,
- properties,
- new ArrayList<>());
+class PaimonLocalFilesSuite extends AnyFunSuite {
+ test("Paimon split read spec carries branch and read-option overrides") {
+ val readOptions = Map("read.batch-size" -> "1").asJava
+ val node = PaimonLocalFilesBuilder.makeNativePaimonLocalFiles(
+ 0,
+ "file:///tmp/paimon-table",
+ "review_branch",
+ readOptions,
+ Seq(Array[Byte](1)).asJava,
+ Seq.empty[String].asJava)
+ val splitReadSpec = node.toProtobuf.getItems(0).getPaimon.getSplitReadSpec
+ assert(splitReadSpec.getBranch == "review_branch")
+ assert(splitReadSpec.getReadOptionsMap == readOptions)
+ assert(splitReadSpec.getSerializedDataSplit.toByteArray.toSeq ==
Seq[Byte](1))
}
}
diff --git
a/gluten-paimon/src-paimon/main/java/org/apache/gluten/substrait/rel/PaimonLocalFilesBuilder.java
b/gluten-ut/spark35/src/test/backends-clickhouse/org/apache/gluten/component/CHPaimonComponentSuite.scala
similarity index 55%
copy from
gluten-paimon/src-paimon/main/java/org/apache/gluten/substrait/rel/PaimonLocalFilesBuilder.java
copy to
gluten-ut/spark35/src/test/backends-clickhouse/org/apache/gluten/component/CHPaimonComponentSuite.scala
index 4d6c33baee..67729c5aff 100644
---
a/gluten-paimon/src-paimon/main/java/org/apache/gluten/substrait/rel/PaimonLocalFilesBuilder.java
+++
b/gluten-ut/spark35/src/test/backends-clickhouse/org/apache/gluten/component/CHPaimonComponentSuite.scala
@@ -14,29 +14,17 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.gluten.substrait.rel;
+package org.apache.gluten.component
-import java.util.List;
-import java.util.Map;
+import org.scalatest.funsuite.AnyFunSuite
-public class PaimonLocalFilesBuilder {
- public static PaimonLocalFilesNode makePaimonLocalFiles(
- Integer index,
- List<String> paths,
- List<Long> starts,
- List<Long> lengths,
- List<Map<String, String>> partitionColumns,
- LocalFilesNode.ReadFileFormat fileFormat,
- List<String> preferredLocations,
- Map<String, String> properties) {
- return new PaimonLocalFilesNode(
- index,
- paths,
- starts,
- lengths,
- partitionColumns,
- fileFormat,
- preferredLocations,
- properties);
+class CHPaimonComponentSuite extends AnyFunSuite {
+ test("Paimon component declares the ClickHouse backend dependency") {
+ val loader = Thread.currentThread().getContextClassLoader
+ val component = "org.apache.gluten.component.CHPaimonComponent"
+ assume(loader.getResource(s"META-INF/gluten-components/$component") !=
null)
+ assert(loader.loadClass(component).getDeclaredConstructor().newInstance()
+ .asInstanceOf[Component].dependencies().contains(
+ classOf[org.apache.gluten.backendsapi.clickhouse.CHBackend]))
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]