This is an automated email from the ASF dual-hosted git repository.
kirs pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/incubator-seatunnel.git
The following commit(s) were added to refs/heads/dev by this push:
new 7870ec1bd [Connector-V2-e2e] Add file connector e2e test cases (#2171)
(#2290)
7870ec1bd is described below
commit 7870ec1bd757ea3eb214dc52c1290db87ba02ede
Author: TyrantLucifer <[email protected]>
AuthorDate: Thu Aug 4 11:18:53 2022 +0800
[Connector-V2-e2e] Add file connector e2e test cases (#2171) (#2290)
* [Connector-V2-e2e] Add file connector e2e test cases (#2171)
* [Connector-V2] Add some comment in file connector e2e test cases (#2171)
* [Bug][Connector-V2] Fix avro conflicts (#2294)
* [Bug][Connector-V2] Fix avro conflicts (#2289)
* [Bug][Connector-V2] Fix avro conflicts (apache#2289)
* [Bug][Connector-V2] Fix dependency error (#2289)
* [Bug][Flink-connector-v2-e2e] Replace the flink docker name (#2291)
(#2297)
* [Improve][UT] Update junit version to 5.x
---
.../e2e/flink/v2/file/FakeSourceToFileIT.java | 35 +++++++++++++++++--
...o_file.conf => fakesource_to_hdfs_parquet.conf} | 26 ++++++++-------
.../resources/file/fakesource_to_hdfs_text.conf} | 26 +++++++--------
..._file.conf => fakesource_to_local_parquet.conf} | 24 +++++++------
..._to_file.conf => fakesource_to_local_text.conf} | 18 +++++-----
.../e2e/spark/v2/file/FakeSourceToFileIT.java | 34 +++++++++++++++++--
.../file/fakesource_to_hdfs_parquet.conf} | 39 +++++++++++-----------
.../resources/file/fakesource_to_hdfs_text.conf} | 37 ++++++++++----------
.../file/fakesource_to_local_parquet.conf} | 37 ++++++++++----------
.../resources/file/fakesource_to_local_text.conf} | 31 ++++++++---------
10 files changed, 188 insertions(+), 119 deletions(-)
diff --git
a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/java/org/apache/seatunnel/e2e/flink/v2/file/FakeSourceToFileIT.java
b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/java/org/apache/seatunnel/e2e/flink/v2/file/FakeSourceToFileIT.java
index 64f2e7cab..9ac33002e 100644
---
a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/java/org/apache/seatunnel/e2e/flink/v2/file/FakeSourceToFileIT.java
+++
b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/java/org/apache/seatunnel/e2e/flink/v2/file/FakeSourceToFileIT.java
@@ -26,9 +26,40 @@ import org.testcontainers.containers.Container;
import java.io.IOException;
public class FakeSourceToFileIT extends FlinkContainer {
+
+ /**
+ * fake source -> local text file sink
+ */
+ @Test
+ public void testFakeSourceToLocalFileText() throws IOException,
InterruptedException {
+ Container.ExecResult execResult =
executeSeaTunnelFlinkJob("/file/fakesource_to_local_text.conf");
+ Assertions.assertEquals(0, execResult.getExitCode());
+ }
+
+ /**
+ * fake source -> local parquet file sink
+ */
+ @Test
+ public void testFakeSourceToLocalFileParquet() throws IOException,
InterruptedException {
+ Container.ExecResult execResult =
executeSeaTunnelFlinkJob("/file/fakesource_to_local_parquet.conf");
+ Assertions.assertEquals(0, execResult.getExitCode());
+ }
+
+ /**
+ * fake source -> hdfs text file sink
+ */
+ @Test
+ public void testFakeSourceToHdfsFileText() throws IOException,
InterruptedException {
+ Container.ExecResult execResult =
executeSeaTunnelFlinkJob("/file/fakesource_to_hdfs_text.conf");
+ Assertions.assertEquals(0, execResult.getExitCode());
+ }
+
+ /**
+ * fake source -> hdfs parquet file sink
+ */
@Test
- public void testFakeSourceToFileSink() throws IOException,
InterruptedException {
- Container.ExecResult execResult =
executeSeaTunnelFlinkJob("/file/fakesource_to_file.conf");
+ public void testFakeSourceToHdfsFileParquet() throws IOException,
InterruptedException {
+ Container.ExecResult execResult =
executeSeaTunnelFlinkJob("/file/fakesource_to_hdfs_parquet.conf");
Assertions.assertEquals(0, execResult.getExitCode());
}
}
diff --git
a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_file.conf
b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_parquet.conf
similarity index 77%
copy from
seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_file.conf
copy to
seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_parquet.conf
index 7e01a805f..9f5fd0b17 100644
---
a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_file.conf
+++
b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_parquet.conf
@@ -27,38 +27,40 @@ env {
}
source {
- # This is a example source plugin **only for test and demonstrate the
feature source plugin**
- FakeSource {
- result_table_name = "fake"
- field_name = "name,age"
- }
+ FakeSource {
+ result_table_name = "fake"
+ field_name = "name,age"
+ }
# If you would like to get more information about how to configure seatunnel
and see full list of source plugins,
- # please go to
https://seatunnel.apache.org/docs/flink/configuration/source-plugins/Fake
+ # please go to https://seatunnel.apache.org/docs/connector-v2/source/Fake
}
transform {
+ sql {
+ sql = "select name,age from fake"
+ }
+
# If you would like to get more information about how to configure seatunnel
and see full list of transform plugins,
- # please go to
https://seatunnel.apache.org/docs/flink/configuration/transform-plugins/Sql
+ # please go to https://seatunnel.apache.org/docs/transform/sql
}
sink {
- LocalFile {
- path="file:///tmp/hive/warehouse/test2"
+ HdfsFile {
+ path="/tmp/hive/warehouse/test2"
field_delimiter="\t"
row_delimiter="\n"
partition_by=["age"]
partition_dir_expression="${k0}=${v0}"
is_partition_field_write_in_file=true
file_name_expression="${transactionId}_${now}"
- file_format="text"
+ file_format="parquet"
sink_columns=["name","age"]
filename_time_format="yyyy.MM.dd"
is_enable_transaction=true
save_mode="error"
-
}
# If you would like to get more information about how to configure seatunnel
and see full list of sink plugins,
- # please go to
https://seatunnel.apache.org/docs/flink/configuration/sink-plugins/Console
+ # please go to https://seatunnel.apache.org/docs/connector-v2/sink/File
}
\ No newline at end of file
diff --git
a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_file.conf
b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_text.conf
similarity index 76%
rename from
seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_file.conf
rename to
seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_text.conf
index e70490855..ef83dfd4e 100644
---
a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_file.conf
+++
b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_text.conf
@@ -27,28 +27,27 @@ env {
}
source {
- # This is a example source plugin **only for test and demonstrate the
feature source plugin**
- FakeSource {
- result_table_name = "fake"
- field_name = "name,age"
- }
+ FakeSource {
+ result_table_name = "fake"
+ field_name = "name,age"
+ }
# If you would like to get more information about how to configure seatunnel
and see full list of source plugins,
- # please go to
https://seatunnel.apache.org/docs/flink/configuration/source-plugins/Fake
+ # please go to https://seatunnel.apache.org/docs/connector-v2/source/Fake
}
transform {
+ sql {
+ sql = "select name,age from fake"
+ }
- sql {
- sql = "select name,age from fake"
- }
# If you would like to get more information about how to configure seatunnel
and see full list of transform plugins,
- # please go to
https://seatunnel.apache.org/docs/flink/configuration/transform-plugins/Sql
+ # please go to https://seatunnel.apache.org/docs/transform/sql
}
sink {
- LocalFile {
- path="file:///tmp/hive/warehouse/test2"
+ HdfsFile {
+ path="/tmp/hive/warehouse/test2"
field_delimiter="\t"
row_delimiter="\n"
partition_by=["age"]
@@ -60,9 +59,8 @@ sink {
filename_time_format="yyyy.MM.dd"
is_enable_transaction=true
save_mode="error"
-
}
# If you would like to get more information about how to configure seatunnel
and see full list of sink plugins,
- # please go to
https://seatunnel.apache.org/docs/flink/configuration/sink-plugins/Console
+ # please go to https://seatunnel.apache.org/docs/connector-v2/sink/File
}
\ No newline at end of file
diff --git
a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_file.conf
b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_local_parquet.conf
similarity index 77%
copy from
seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_file.conf
copy to
seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_local_parquet.conf
index 7e01a805f..9e2d5ad96 100644
---
a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_file.conf
+++
b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_local_parquet.conf
@@ -27,38 +27,40 @@ env {
}
source {
- # This is a example source plugin **only for test and demonstrate the
feature source plugin**
- FakeSource {
- result_table_name = "fake"
- field_name = "name,age"
- }
+ FakeSource {
+ result_table_name = "fake"
+ field_name = "name,age"
+ }
# If you would like to get more information about how to configure seatunnel
and see full list of source plugins,
- # please go to
https://seatunnel.apache.org/docs/flink/configuration/source-plugins/Fake
+ # please go to https://seatunnel.apache.org/docs/connector-v2/source/Fake
}
transform {
+ sql {
+ sql = "select name,age from fake"
+ }
+
# If you would like to get more information about how to configure seatunnel
and see full list of transform plugins,
- # please go to
https://seatunnel.apache.org/docs/flink/configuration/transform-plugins/Sql
+ # please go to https://seatunnel.apache.org/docs/transform/sql
}
sink {
LocalFile {
- path="file:///tmp/hive/warehouse/test2"
+ path="/tmp/hive/warehouse/test2"
field_delimiter="\t"
row_delimiter="\n"
partition_by=["age"]
partition_dir_expression="${k0}=${v0}"
is_partition_field_write_in_file=true
file_name_expression="${transactionId}_${now}"
- file_format="text"
+ file_format="parquet"
sink_columns=["name","age"]
filename_time_format="yyyy.MM.dd"
is_enable_transaction=true
save_mode="error"
-
}
# If you would like to get more information about how to configure seatunnel
and see full list of sink plugins,
- # please go to
https://seatunnel.apache.org/docs/flink/configuration/sink-plugins/Console
+ # please go to https://seatunnel.apache.org/docs/connector-v2/sink/File
}
\ No newline at end of file
diff --git
a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_file.conf
b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_local_text.conf
similarity index 86%
copy from
seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_file.conf
copy to
seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_local_text.conf
index 7e01a805f..d162b101f 100644
---
a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_file.conf
+++
b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_local_text.conf
@@ -27,24 +27,27 @@ env {
}
source {
- # This is a example source plugin **only for test and demonstrate the
feature source plugin**
- FakeSource {
- result_table_name = "fake"
- field_name = "name,age"
- }
+ FakeSource {
+ result_table_name = "fake"
+ field_name = "name,age"
+ }
# If you would like to get more information about how to configure seatunnel
and see full list of source plugins,
# please go to
https://seatunnel.apache.org/docs/flink/configuration/source-plugins/Fake
}
transform {
+ sql {
+ sql = "select name,age from fake"
+ }
+
# If you would like to get more information about how to configure seatunnel
and see full list of transform plugins,
# please go to
https://seatunnel.apache.org/docs/flink/configuration/transform-plugins/Sql
}
sink {
LocalFile {
- path="file:///tmp/hive/warehouse/test2"
+ path="/tmp/hive/warehouse/test2"
field_delimiter="\t"
row_delimiter="\n"
partition_by=["age"]
@@ -56,9 +59,8 @@ sink {
filename_time_format="yyyy.MM.dd"
is_enable_transaction=true
save_mode="error"
-
}
# If you would like to get more information about how to configure seatunnel
and see full list of sink plugins,
- # please go to
https://seatunnel.apache.org/docs/flink/configuration/sink-plugins/Console
+ # please go to https://seatunnel.apache.org/docs/connector-v2/sink/File
}
\ No newline at end of file
diff --git
a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/java/org/apache/seatunnel/e2e/spark/v2/file/FakeSourceToFileIT.java
b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/java/org/apache/seatunnel/e2e/spark/v2/file/FakeSourceToFileIT.java
index 24a4472c1..fe48fb974 100644
---
a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/java/org/apache/seatunnel/e2e/spark/v2/file/FakeSourceToFileIT.java
+++
b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/java/org/apache/seatunnel/e2e/spark/v2/file/FakeSourceToFileIT.java
@@ -31,9 +31,39 @@ import java.io.IOException;
*/
public class FakeSourceToFileIT extends SparkContainer {
+ /**
+ * fake source -> local text file sink
+ */
@Test
- public void testFakeSourceToFile() throws IOException,
InterruptedException {
- Container.ExecResult execResult =
executeSeaTunnelSparkJob("/file/fakesource_to_file.conf");
+ public void testFakeSourceToLocalFileText() throws IOException,
InterruptedException {
+ Container.ExecResult execResult =
executeSeaTunnelSparkJob("/file/fakesource_to_local_text.conf");
+ Assertions.assertEquals(0, execResult.getExitCode());
+ }
+
+ /**
+ * fake source -> local parquet file sink
+ */
+ @Test
+ public void testFakeSourceToLocalFileParquet() throws IOException,
InterruptedException {
+ Container.ExecResult execResult =
executeSeaTunnelSparkJob("/file/fakesource_to_local_parquet.conf");
+ Assertions.assertEquals(0, execResult.getExitCode());
+ }
+
+ /**
+ * fake source -> hdfs text file sink
+ */
+ @Test
+ public void testFakeSourceToHdfsFileText() throws IOException,
InterruptedException {
+ Container.ExecResult execResult =
executeSeaTunnelSparkJob("/file/fakesource_to_hdfs_text.conf");
+ Assertions.assertEquals(0, execResult.getExitCode());
+ }
+
+ /**
+ * fake source -> hdfs parquet file sink
+ */
+ @Test
+ public void testFakeSourceToHdfsFileParquet() throws IOException,
InterruptedException {
+ Container.ExecResult execResult =
executeSeaTunnelSparkJob("/file/fakesource_to_hdfs_parquet.conf");
Assertions.assertEquals(0, execResult.getExitCode());
}
}
diff --git
a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_file.conf
b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_parquet.conf
similarity index 65%
copy from
seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_file.conf
copy to
seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_parquet.conf
index 7e01a805f..bdae80d74 100644
---
a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_file.conf
+++
b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_parquet.conf
@@ -14,51 +14,52 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-######
-###### This config file is a demonstration of streaming processing in
seatunnel config
-######
env {
- # You can set flink configuration here
- execution.parallelism = 1
+ # You can set spark configuration here
+ spark.app.name = "SeaTunnel"
+ spark.executor.instances = 2
+ spark.executor.cores = 1
+ spark.executor.memory = "1g"
+ spark.master = local
job.mode = "BATCH"
- #execution.checkpoint.interval = 10000
- #execution.checkpoint.data-uri = "hdfs://localhost:9000/checkpoint"
}
source {
- # This is a example source plugin **only for test and demonstrate the
feature source plugin**
- FakeSource {
- result_table_name = "fake"
- field_name = "name,age"
- }
+ FakeSource {
+ result_table_name = "fake"
+ field_name = "name,age"
+ }
# If you would like to get more information about how to configure seatunnel
and see full list of source plugins,
- # please go to
https://seatunnel.apache.org/docs/flink/configuration/source-plugins/Fake
+ # please go to https://seatunnel.apache.org/docs/connector-v2/source/Fake
}
transform {
+ sql {
+ sql = "select name,age from fake"
+ }
+
# If you would like to get more information about how to configure seatunnel
and see full list of transform plugins,
- # please go to
https://seatunnel.apache.org/docs/flink/configuration/transform-plugins/Sql
+ # please go to https://seatunnel.apache.org/docs/transform/sql
}
sink {
- LocalFile {
- path="file:///tmp/hive/warehouse/test2"
+ HdfsFile {
+ path="/tmp/hive/warehouse/test2"
field_delimiter="\t"
row_delimiter="\n"
partition_by=["age"]
partition_dir_expression="${k0}=${v0}"
is_partition_field_write_in_file=true
file_name_expression="${transactionId}_${now}"
- file_format="text"
+ file_format="parquet"
sink_columns=["name","age"]
filename_time_format="yyyy.MM.dd"
is_enable_transaction=true
save_mode="error"
-
}
# If you would like to get more information about how to configure seatunnel
and see full list of sink plugins,
- # please go to
https://seatunnel.apache.org/docs/flink/configuration/sink-plugins/Console
+ # please go to https://seatunnel.apache.org/docs/connector-v2/sink/File
}
\ No newline at end of file
diff --git
a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_file.conf
b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_text.conf
similarity index 66%
copy from
seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_file.conf
copy to
seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_text.conf
index 7e01a805f..b682d3831 100644
---
a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_file.conf
+++
b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_text.conf
@@ -14,37 +14,39 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-######
-###### This config file is a demonstration of streaming processing in
seatunnel config
-######
env {
- # You can set flink configuration here
- execution.parallelism = 1
+ # You can set spark configuration here
+ spark.app.name = "SeaTunnel"
+ spark.executor.instances = 2
+ spark.executor.cores = 1
+ spark.executor.memory = "1g"
+ spark.master = local
job.mode = "BATCH"
- #execution.checkpoint.interval = 10000
- #execution.checkpoint.data-uri = "hdfs://localhost:9000/checkpoint"
}
source {
- # This is a example source plugin **only for test and demonstrate the
feature source plugin**
- FakeSource {
- result_table_name = "fake"
- field_name = "name,age"
- }
+ FakeSource {
+ result_table_name = "fake"
+ field_name = "name,age"
+ }
# If you would like to get more information about how to configure seatunnel
and see full list of source plugins,
- # please go to
https://seatunnel.apache.org/docs/flink/configuration/source-plugins/Fake
+ # please go to https://seatunnel.apache.org/docs/connector-v2/source/Fake
}
transform {
+ sql {
+ sql = "select name,age from fake"
+ }
+
# If you would like to get more information about how to configure seatunnel
and see full list of transform plugins,
- # please go to
https://seatunnel.apache.org/docs/flink/configuration/transform-plugins/Sql
+ # please go to https://seatunnel.apache.org/docs/transform/sql
}
sink {
- LocalFile {
- path="file:///tmp/hive/warehouse/test2"
+ HdfsFile {
+ path="/tmp/hive/warehouse/test2"
field_delimiter="\t"
row_delimiter="\n"
partition_by=["age"]
@@ -56,9 +58,8 @@ sink {
filename_time_format="yyyy.MM.dd"
is_enable_transaction=true
save_mode="error"
-
}
# If you would like to get more information about how to configure seatunnel
and see full list of sink plugins,
- # please go to
https://seatunnel.apache.org/docs/flink/configuration/sink-plugins/Console
+ # please go to https://seatunnel.apache.org/docs/connector-v2/sink/File
}
\ No newline at end of file
diff --git
a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_file.conf
b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_local_parquet.conf
similarity index 65%
copy from
seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_file.conf
copy to
seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_local_parquet.conf
index 7e01a805f..b5d141212 100644
---
a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_file.conf
+++
b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_local_parquet.conf
@@ -14,51 +14,52 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-######
-###### This config file is a demonstration of streaming processing in
seatunnel config
-######
env {
- # You can set flink configuration here
- execution.parallelism = 1
+ # You can set spark configuration here
+ spark.app.name = "SeaTunnel"
+ spark.executor.instances = 2
+ spark.executor.cores = 1
+ spark.executor.memory = "1g"
+ spark.master = local
job.mode = "BATCH"
- #execution.checkpoint.interval = 10000
- #execution.checkpoint.data-uri = "hdfs://localhost:9000/checkpoint"
}
source {
- # This is a example source plugin **only for test and demonstrate the
feature source plugin**
- FakeSource {
- result_table_name = "fake"
- field_name = "name,age"
- }
+ FakeSource {
+ result_table_name = "fake"
+ field_name = "name,age"
+ }
# If you would like to get more information about how to configure seatunnel
and see full list of source plugins,
- # please go to
https://seatunnel.apache.org/docs/flink/configuration/source-plugins/Fake
+ # please go to https://seatunnel.apache.org/docs/connector-v2/source/Fake
}
transform {
+ sql {
+ sql = "select name,age from fake"
+ }
+
# If you would like to get more information about how to configure seatunnel
and see full list of transform plugins,
- # please go to
https://seatunnel.apache.org/docs/flink/configuration/transform-plugins/Sql
+ # please go to https://seatunnel.apache.org/docs/transform/sql
}
sink {
LocalFile {
- path="file:///tmp/hive/warehouse/test2"
+ path="/tmp/hive/warehouse/test2"
field_delimiter="\t"
row_delimiter="\n"
partition_by=["age"]
partition_dir_expression="${k0}=${v0}"
is_partition_field_write_in_file=true
file_name_expression="${transactionId}_${now}"
- file_format="text"
+ file_format="parquet"
sink_columns=["name","age"]
filename_time_format="yyyy.MM.dd"
is_enable_transaction=true
save_mode="error"
-
}
# If you would like to get more information about how to configure seatunnel
and see full list of sink plugins,
- # please go to
https://seatunnel.apache.org/docs/flink/configuration/sink-plugins/Console
+ # please go to https://seatunnel.apache.org/docs/connector-v2/sink/File
}
\ No newline at end of file
diff --git
a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_file.conf
b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_local_text.conf
similarity index 74%
rename from
seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_file.conf
rename to
seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_local_text.conf
index 7e01a805f..733a48e61 100644
---
a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_file.conf
+++
b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_local_text.conf
@@ -14,37 +14,39 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-######
-###### This config file is a demonstration of streaming processing in
seatunnel config
-######
env {
- # You can set flink configuration here
- execution.parallelism = 1
+ # You can set spark configuration here
+ spark.app.name = "SeaTunnel"
+ spark.executor.instances = 2
+ spark.executor.cores = 1
+ spark.executor.memory = "1g"
+ spark.master = local
job.mode = "BATCH"
- #execution.checkpoint.interval = 10000
- #execution.checkpoint.data-uri = "hdfs://localhost:9000/checkpoint"
}
source {
- # This is a example source plugin **only for test and demonstrate the
feature source plugin**
- FakeSource {
- result_table_name = "fake"
- field_name = "name,age"
- }
+ FakeSource {
+ result_table_name = "fake"
+ field_name = "name,age"
+ }
# If you would like to get more information about how to configure seatunnel
and see full list of source plugins,
# please go to
https://seatunnel.apache.org/docs/flink/configuration/source-plugins/Fake
}
transform {
+ sql {
+ sql = "select name,age from fake"
+ }
+
# If you would like to get more information about how to configure seatunnel
and see full list of transform plugins,
# please go to
https://seatunnel.apache.org/docs/flink/configuration/transform-plugins/Sql
}
sink {
LocalFile {
- path="file:///tmp/hive/warehouse/test2"
+ path="/tmp/hive/warehouse/test2"
field_delimiter="\t"
row_delimiter="\n"
partition_by=["age"]
@@ -56,9 +58,8 @@ sink {
filename_time_format="yyyy.MM.dd"
is_enable_transaction=true
save_mode="error"
-
}
# If you would like to get more information about how to configure seatunnel
and see full list of sink plugins,
- # please go to
https://seatunnel.apache.org/docs/flink/configuration/sink-plugins/Console
+ # please go to https://seatunnel.apache.org/docs/connector-v2/sink/File
}
\ No newline at end of file
