This is an automated email from the ASF dual-hosted git repository.
kirs pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/incubator-seatunnel.git
The following commit(s) were added to refs/heads/dev by this push:
new 9158643 [Feature][seatunnel-examples] spark local environment run
quickly and debug locally developed code easily (#993)
9158643 is described below
commit 9158643ea9728c931be9fabb49f7a7ee1f101cba
Author: felix.wang <[email protected]>
AuthorDate: Tue Jan 11 09:52:46 2022 +0800
[Feature][seatunnel-examples] spark local environment run quickly and debug
locally developed code easily (#993)
* add spark local exmaple
* add license
* Optimize package nameļ¼and class name
---
seatunnel-examples/pom.xml | 1 +
.../flink/LocalFlinkExample.java} | 4 +-
.../seatunnel-spark-examples/pom.xml | 74 ++++++++++++++++++++
.../example/spark/LocalSparkExample.java} | 12 ++--
.../resources/examples/spark.batch.conf.template | 80 ++++++++++++++++++++++
5 files changed, 163 insertions(+), 8 deletions(-)
diff --git a/seatunnel-examples/pom.xml b/seatunnel-examples/pom.xml
index 4edb908..15e2000 100644
--- a/seatunnel-examples/pom.xml
+++ b/seatunnel-examples/pom.xml
@@ -31,5 +31,6 @@
<packaging>pom</packaging>
<modules>
<module>seatunnel-flink-examples</module>
+ <module>seatunnel-spark-examples</module>
</modules>
</project>
\ No newline at end of file
diff --git
a/seatunnel-examples/seatunnel-flink-examples/src/main/java/org/apache/seatunnel/test/flink/LocalTest.java
b/seatunnel-examples/seatunnel-flink-examples/src/main/java/org/apache/seatunnel/example/flink/LocalFlinkExample.java
similarity index 95%
copy from
seatunnel-examples/seatunnel-flink-examples/src/main/java/org/apache/seatunnel/test/flink/LocalTest.java
copy to
seatunnel-examples/seatunnel-flink-examples/src/main/java/org/apache/seatunnel/example/flink/LocalFlinkExample.java
index eab9cfc..8675df7 100644
---
a/seatunnel-examples/seatunnel-flink-examples/src/main/java/org/apache/seatunnel/test/flink/LocalTest.java
+++
b/seatunnel-examples/seatunnel-flink-examples/src/main/java/org/apache/seatunnel/example/flink/LocalFlinkExample.java
@@ -15,14 +15,14 @@
* limitations under the License.
*/
-package org.apache.seatunnel.test.flink;
+package org.apache.seatunnel.example.flink;
import static org.apache.seatunnel.utils.Engine.FLINK;
import org.apache.seatunnel.Seatunnel;
import org.apache.seatunnel.config.command.CommandLineArgs;
-public class LocalTest {
+public class LocalFlinkExample {
public static final String TEST_RESOURCE_DIR =
"/seatunnel-examples/seatunnel-flink-examples/src/main/resources/examples/";
diff --git a/seatunnel-examples/seatunnel-spark-examples/pom.xml
b/seatunnel-examples/seatunnel-spark-examples/pom.xml
new file mode 100644
index 0000000..e6b2b30
--- /dev/null
+++ b/seatunnel-examples/seatunnel-spark-examples/pom.xml
@@ -0,0 +1,74 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0";
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd";>
+ <parent>
+ <artifactId>seatunnel-examples</artifactId>
+ <groupId>org.apache.seatunnel</groupId>
+ <version>2.0.5-SNAPSHOT</version>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>seatunnel-spark-examples</artifactId>
+ <properties>
+ <spark.scope>compile</spark.scope>
+ </properties>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.seatunnel</groupId>
+ <artifactId>seatunnel-core-spark</artifactId>
+ <version>${project.parent.version}</version>
+ </dependency>
+ <!--spark-->
+ <dependency>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-streaming_${scala.binary.version}</artifactId>
+ <version>${spark.version}</version>
+ <scope>${spark.scope}</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-core_${scala.binary.version}</artifactId>
+ <version>${spark.version}</version>
+ <scope>${spark.scope}</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-sql_${scala.binary.version}</artifactId>
+ <version>${spark.version}</version>
+ <scope>${spark.scope}</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-hive_${scala.binary.version}</artifactId>
+ <version>${spark.version}</version>
+ <scope>${spark.scope}</scope>
+ </dependency>
+ <dependency>
+ <groupId>net.jpountz.lz4</groupId>
+ <artifactId>lz4</artifactId>
+ <version>1.3.0</version>
+ </dependency>
+ </dependencies>
+</project>
\ No newline at end of file
diff --git
a/seatunnel-examples/seatunnel-flink-examples/src/main/java/org/apache/seatunnel/test/flink/LocalTest.java
b/seatunnel-examples/seatunnel-spark-examples/src/main/java/org/apache/seatunnel/example/spark/LocalSparkExample.java
similarity index 79%
rename from
seatunnel-examples/seatunnel-flink-examples/src/main/java/org/apache/seatunnel/test/flink/LocalTest.java
rename to
seatunnel-examples/seatunnel-spark-examples/src/main/java/org/apache/seatunnel/example/spark/LocalSparkExample.java
index eab9cfc..33c2b3a 100644
---
a/seatunnel-examples/seatunnel-flink-examples/src/main/java/org/apache/seatunnel/test/flink/LocalTest.java
+++
b/seatunnel-examples/seatunnel-spark-examples/src/main/java/org/apache/seatunnel/example/spark/LocalSparkExample.java
@@ -15,21 +15,21 @@
* limitations under the License.
*/
-package org.apache.seatunnel.test.flink;
+package org.apache.seatunnel.example.spark;
-import static org.apache.seatunnel.utils.Engine.FLINK;
+import static org.apache.seatunnel.utils.Engine.SPARK;
import org.apache.seatunnel.Seatunnel;
import org.apache.seatunnel.config.command.CommandLineArgs;
-public class LocalTest {
+public class LocalSparkExample {
- public static final String TEST_RESOURCE_DIR =
"/seatunnel-examples/seatunnel-flink-examples/src/main/resources/examples/";
+ public static final String TEST_RESOURCE_DIR =
"/seatunnel-examples/seatunnel-spark-examples/src/main/resources/examples/";
public static void main(String[] args) {
- String configFile = getTestConfigFile("fake_to_console.conf");
+ String configFile = getTestConfigFile("spark.batch.conf.template");
CommandLineArgs flinkArgs = new CommandLineArgs(configFile, false);
- Seatunnel.run(flinkArgs, FLINK, args);
+ Seatunnel.run(flinkArgs, SPARK, args);
}
public static String getTestConfigFile(String configFile) {
diff --git
a/seatunnel-examples/seatunnel-spark-examples/src/main/resources/examples/spark.batch.conf.template
b/seatunnel-examples/seatunnel-spark-examples/src/main/resources/examples/spark.batch.conf.template
new file mode 100644
index 0000000..cdb7438
--- /dev/null
+++
b/seatunnel-examples/seatunnel-spark-examples/src/main/resources/examples/spark.batch.conf.template
@@ -0,0 +1,80 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+######
+###### This config file is a demonstration of batch processing in SeaTunnel
config
+######
+
+env {
+ # You can set spark configuration here
+ # see available properties defined by spark:
https://spark.apache.org/docs/latest/configuration.html#available-properties
+ spark.app.name = "SeaTunnel"
+ spark.executor.instances = 2
+ spark.executor.cores = 1
+ spark.executor.memory = "1g"
+ spark.master = local
+}
+
+source {
+ # This is a example input plugin **only for test and demonstrate the feature
input plugin**
+ Fake {
+ result_table_name = "my_dataset"
+ }
+
+
+
+ # You can also use other input plugins, such as hdfs
+ # hdfs {
+ # result_table_name = "accesslog"
+ # path = "hdfs://hadoop-cluster-01/nginx/accesslog"
+ # format = "json"
+ # }
+
+ # If you would like to get more information about how to configure seatunnel
and see full list of input plugins,
+ # please go to
https://interestinglab.github.io/seatunnel-docs/#/zh-cn/v2/spark/developing-plugin
+}
+
+transform {
+ # split data by specific delimiter
+
+
+
+
+ # you can also you other filter plugins, such as sql
+ # sql {
+ # sql = "select * from accesslog where request_time > 1000"
+ # }
+
+ # If you would like to get more information about how to configure seatunnel
and see full list of filter plugins,
+ # please go to
https://interestinglab.github.io/seatunnel-docs/#/zh-cn/v2/spark/developing-plugin
+}
+
+sink {
+ # choose stdout output plugin to output data to console
+ Console {}
+
+
+
+ # you can also you other output plugins, such as sql
+ # hdfs {
+ # path = "hdfs://hadoop-cluster-01/nginx/accesslog_processed"
+ # save_mode = "append"
+ # }
+
+ # If you would like to get more information about how to configure seatunnel
and see full list of output plugins,
+ # please go to
https://interestinglab.github.io/seatunnel-docs/#/zh-cn/v2/spark/developing-plugin
+}
