This is an automated email from the ASF dual-hosted git repository.
diwu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris-spark-connector.git
The following commit(s) were added to refs/heads/master by this push:
new a2c998e [test](example) add example for spark write and read (#310)
a2c998e is described below
commit a2c998e64790280acc726ca116d5c3db9febda96
Author: wudi <[email protected]>
AuthorDate: Wed Apr 16 17:28:09 2025 +0800
[test](example) add example for spark write and read (#310)
---
.../doris/spark/example/DorisReadExample.scala | 38 ++++++++++++++
.../spark/example/DorisWriteBatchExample.scala | 45 +++++++++++++++++
.../spark/example/DorisWriteStreamExample.scala | 59 ++++++++++++++++++++++
3 files changed, 142 insertions(+)
diff --git
a/spark-doris-connector/spark-doris-connector-it/src/test/java/org/apache/doris/spark/example/DorisReadExample.scala
b/spark-doris-connector/spark-doris-connector-it/src/test/java/org/apache/doris/spark/example/DorisReadExample.scala
new file mode 100644
index 0000000..2b8d5ed
--- /dev/null
+++
b/spark-doris-connector/spark-doris-connector-it/src/test/java/org/apache/doris/spark/example/DorisReadExample.scala
@@ -0,0 +1,38 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.spark.example
+
+import org.apache.spark.sql.SparkSession
+
+/**
+ * Example to read data from Doris using Spark Batch.
+ */
+object DorisReadExample {
+
+ def main(args: Array[String]): Unit = {
+ val session = SparkSession.builder().master("local[1]").getOrCreate()
+ val dorisSparkDF = session.read.format("doris")
+ .option("doris.table.identifier", "test.student")
+ .option("doris.fenodes", "127.0.0.1:8030")
+ .option("user", "root")
+ .option("password", "")
+ .load()
+
+ dorisSparkDF.show()
+ }
+}
diff --git
a/spark-doris-connector/spark-doris-connector-it/src/test/java/org/apache/doris/spark/example/DorisWriteBatchExample.scala
b/spark-doris-connector/spark-doris-connector-it/src/test/java/org/apache/doris/spark/example/DorisWriteBatchExample.scala
new file mode 100644
index 0000000..cd35e2a
--- /dev/null
+++
b/spark-doris-connector/spark-doris-connector-it/src/test/java/org/apache/doris/spark/example/DorisWriteBatchExample.scala
@@ -0,0 +1,45 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.spark.example
+
+import org.apache.spark.sql.SparkSession
+
+/**
+ * Example to write data to Doris using Spark Batch.
+ */
+object DorisWriteBatchExample {
+
+ def main(args: Array[String]): Unit = {
+ val session = SparkSession.builder().master("local[1]").getOrCreate()
+ val mockDataDF = session.createDataFrame(Seq(
+ ("1", "doris", "18"),
+ ("2", "apache", "28"),
+ ("3", "spark", "78")
+ )).toDF("id", "name","age")
+
+ mockDataDF.show()
+ mockDataDF.write.format("doris")
+ .option("doris.table.identifier", "test.student")
+ .option("doris.fenodes", "127.0.0.1:8030")
+ .option("user", "root")
+ .option("password", "")
+ .mode("append")
+ .save()
+ session.stop()
+ }
+}
diff --git
a/spark-doris-connector/spark-doris-connector-it/src/test/java/org/apache/doris/spark/example/DorisWriteStreamExample.scala
b/spark-doris-connector/spark-doris-connector-it/src/test/java/org/apache/doris/spark/example/DorisWriteStreamExample.scala
new file mode 100644
index 0000000..b0b153e
--- /dev/null
+++
b/spark-doris-connector/spark-doris-connector-it/src/test/java/org/apache/doris/spark/example/DorisWriteStreamExample.scala
@@ -0,0 +1,59 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.spark.example
+
+import org.apache.spark.sql.SparkSession
+
+/**
+ * Example to write data to Doris using Spark Structured Streaming.
+ */
+object DorisWriteStreamExample {
+
+ def main(args: Array[String]): Unit = {
+
+ val spark = SparkSession.builder()
+ .appName("RateSourceExample")
+ .master("local[1]")
+ .getOrCreate()
+
+ val rateStream = spark.readStream
+ .format("rate")
+ .option("rowsPerSecond", 10)
+ .load()
+
+ /**
+ * root
+ * |-- timestamp: timestamp (nullable = true)
+ * |-- value: long (nullable = true)
+ */
+ rateStream.printSchema();
+
+ rateStream.writeStream
+ .format("doris")
+ .option("checkpointLocation", "/tmp/checkpoint")
+ .option("doris.table.identifier",
"test_doris_streaming.tbl_write_tbl_stream")
+ .option("doris.fenodes", "127.0.0.1:8030")
+ .option("user", "root")
+ .option("password", "")
+ .start()
+ .awaitTermination()
+
+ spark.stop();
+ }
+
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
