This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 636978a9142 [fix](test) remove spark load test cases (#58269)
636978a9142 is described below
commit 636978a9142fa9c086996ed3b077968ff6df8cf2
Author: Mingyu Chen (Rayner) <[email protected]>
AuthorDate: Mon Nov 24 16:01:37 2025 +0800
[fix](test) remove spark load test cases (#58269)
---
.../data/load_p0/spark_load/all_types1.txt | 11 --
.../data/load_p0/spark_load/all_types2.txt | 11 --
.../data/load_p0/spark_load/test_spark_load.out | 37 -----
.../load_p0/spark_load/test_spark_load.groovy | 150 ---------------------
4 files changed, 209 deletions(-)
diff --git a/regression-test/data/load_p0/spark_load/all_types1.txt
b/regression-test/data/load_p0/spark_load/all_types1.txt
deleted file mode 100644
index 3b4e1ad6dc8..00000000000
--- a/regression-test/data/load_p0/spark_load/all_types1.txt
+++ /dev/null
@@ -1,11 +0,0 @@
-10000,aa,北京,0,11,4444,5555555,41232314,3.14,123.3423,111.111,111.111,2017-10-01,2017-10-01,2017-10-01
06:00:00,2017-10-01 06:00:00
-10001,bb,北京,0,22,3333,666,2768658,5.32,123111.3242,222.222,222.222,2017-10-02,2017-10-02,2017-10-02
07:00:00,2017-10-02 07:00:00
-10002,cc,北京,1,33,2222,453,5463456,4.321,11111.23423,333.333,333.333,2017-10-03,2017-10-03,2017-10-03
17:05:45,2017-10-03 17:05:45
-10003,dd,上海,1,44,1111,-3241,-45235,1.34,54626.324,444.444,444.444,2017-10-04,2017-10-04,2017-10-04
12:59:12,2017-10-04 12:59:12
-10004,ee,成都,0,55,-9999,21342,4513456,1.22,111.33,555.555,555.555,2017-10-05,2017-10-05,2017-10-05
11:20:00,2017-10-05 11:20:00
-10005,ff,西安,0,66,8888,64562,4356,9.133,23423.45,666.666,666.666,2017-10-06,2017-10-06,2017-10-06
12:00:15,2017-10-06 12:00:15
-10006,gg,深圳,1,77,-7777,-12313342,34534,8.100,12,777.777,777.777,2017-10-07,2017-10-07,2017-10-07
13:20:22,2017-10-07 13:20:22
-10007,hh,杭州,0,88,6666,314234,43535356,34.124,324,888.888,888.888,2017-10-08,2017-10-08,2017-10-08
14:58:10,2017-10-08 14:58:10
-10008,ii,上海,1,99,-5555,1341,23434534,342.120,34234.1,999.999,999.999,2017-10-09,2017-10-09,2017-10-09
25:12:22,2017-10-09 25:12:22
-10009,jj,南京,0,11,4444,-123,53623567,11.22,324.33,111.111,111.111,2017-10-10,2017-10-10,2017-10-10
16:25:42,2017-10-10 16:25:42
-10010,kk,成都,0,22,-3333,12314,674567,13,45464.435,222.222,222.222,2017-10-11,2017-10-11,2017-10-11
17:22:24,2017-10-11 17:22:24
\ No newline at end of file
diff --git a/regression-test/data/load_p0/spark_load/all_types2.txt
b/regression-test/data/load_p0/spark_load/all_types2.txt
deleted file mode 100644
index 429b1d99d24..00000000000
--- a/regression-test/data/load_p0/spark_load/all_types2.txt
+++ /dev/null
@@ -1,11 +0,0 @@
-10011|aa|北京|0|11|4444|5555555|41232314|3.14|123.3423|111.111|111.111|2017-10-01|2017-10-01|2017-10-01
06:00:00|2017-10-01 06:00:00
-10012|bb|北京|0|22|3333|666|2768658|5.32|123111.3242|222.222|222.222|2017-10-02|2017-10-02|2017-10-02
07:00:00|2017-10-02 07:00:00
-10013|cc|北京|1|33|2222|453|5463456|4.321|11111.23423|333.333|333.333|2017-10-03|2017-10-03|2017-10-03
17:05:45|2017-10-03 17:05:45
-10014|dd|上海|1|44|1111|-3241|-45235|1.34|54626.324|444.444|444.444|2017-10-04|2017-10-04|2017-10-04
12:59:12|2017-10-04 12:59:12
-10015|ee|成都|0|55|-9999|21342|4513456|1.22|111.33|555.555|555.555|2017-10-05|2017-10-05|2017-10-05
11:20:00|2017-10-05 11:20:00
-10016|ff|西安|0|66|8888|64562|4356|9.133|23423.45|666.666|666.666|2017-10-06|2017-10-06|2017-10-06
12:00:15|2017-10-06 12:00:15
-10017|gg|深圳|1|77|-7777|-12313342|34534|8.100|12|777.777|777.777|2017-10-07|2017-10-07|2017-10-07
13:20:22|2017-10-07 13:20:22
-10018|hh|杭州|0|88|6666|314234|43535356|34.124|324|888.888|888.888|2017-10-08|2017-10-08|2017-10-08
14:58:10|2017-10-08 14:58:10
-10019|ii|上海|1|99|-5555|1341|23434534|342.120|34234.1|999.999|999.999|2017-10-09|2017-10-09|2017-10-09
25:12:22|2017-10-09 25:12:22
-10020|jj|南京|0|11|4444|-123|53623567|11.22|324.33|111.111|111.111|2017-10-10|2017-10-10|2017-10-10
16:25:42|2017-10-10 16:25:42
-10021|kk|成都|0|22|-3333|12314|674567|13|45464.435|222.222|222.222|2017-10-11|2017-10-11|2017-10-11
17:22:24|2017-10-11 17:22:24
\ No newline at end of file
diff --git a/regression-test/data/load_p0/spark_load/test_spark_load.out
b/regression-test/data/load_p0/spark_load/test_spark_load.out
deleted file mode 100644
index aa4d662c235..00000000000
--- a/regression-test/data/load_p0/spark_load/test_spark_load.out
+++ /dev/null
@@ -1,37 +0,0 @@
--- This file is automatically generated. You should know what you did if you
want to edit this
--- !select --
-10000 aa 北京 false 11 4444 5555555 41232314 3.14
123.3423 111.111 111.111 2017-10-01 2017-10-01
2017-10-01T06:00 2017-10-01T06:00
-10001 bb 北京 false 22 3333 666 2768658 5.32
123111.3242 222.222 222.222 2017-10-02 2017-10-02
2017-10-02T07:00 2017-10-02T07:00
-10002 cc 北京 true 33 2222 453 5463456 4.321
11111.23423 333.333 333.333 2017-10-03 2017-10-03
2017-10-03T17:05:45 2017-10-03T17:05:45
-10003 dd 上海 true 44 1111 -3241 -45235 1.34
54626.324 444.444 444.444 2017-10-04 2017-10-04
2017-10-04T12:59:12 2017-10-04T12:59:12
-10004 ee 成都 false 55 -9999 21342 4513456 1.22 111.33
555.555 555.555 2017-10-05 2017-10-05 2017-10-05T11:20
2017-10-05T11:20
-10005 ff 西安 false 66 8888 64562 4356 9.133
23423.45 666.666 666.666 2017-10-06 2017-10-06
2017-10-06T12:00:15 2017-10-06T12:00:15
-10006 gg 深圳 true 77 -7777 -12313342 34534 8.1
12.0 777.777 777.777 2017-10-07 2017-10-07 2017-10-07T13:20:22
2017-10-07T13:20:22
-10007 hh 杭州 false 88 6666 314234 43535356 34.124
324.0 888.888 888.888 2017-10-08 2017-10-08 2017-10-08T14:58:10
2017-10-08T14:58:10
-10008 ii 上海 true 99 -5555 1341 23434534 342.12
34234.1 999.999 999.999 2017-10-09 2017-10-09 \N \N
-10009 jj 南京 false 11 4444 -123 53623567 11.22
324.33 111.111 111.111 2017-10-10 2017-10-10 2017-10-10T16:25:42
2017-10-10T16:25:42
-10010 kk 成都 false 22 -3333 12314 674567 13.0
45464.435 222.222 222.222 2017-10-11 2017-10-11
2017-10-11T17:22:24 2017-10-11T17:22:24
-10011 aa 北京 false 11 4444 5555555 41232314 3.14
123.3423 111.111 111.111 2017-10-01 2017-10-01
2017-10-01T06:00 2017-10-01T06:00
-10012 bb 北京 false 22 3333 666 2768658 5.32
123111.3242 222.222 222.222 2017-10-02 2017-10-02
2017-10-02T07:00 2017-10-02T07:00
-10013 cc 北京 true 33 2222 453 5463456 4.321
11111.23423 333.333 333.333 2017-10-03 2017-10-03
2017-10-03T17:05:45 2017-10-03T17:05:45
-10014 dd 上海 true 44 1111 -3241 -45235 1.34
54626.324 444.444 444.444 2017-10-04 2017-10-04
2017-10-04T12:59:12 2017-10-04T12:59:12
-10015 ee 成都 false 55 -9999 21342 4513456 1.22 111.33
555.555 555.555 2017-10-05 2017-10-05 2017-10-05T11:20
2017-10-05T11:20
-10016 ff 西安 false 66 8888 64562 4356 9.133
23423.45 666.666 666.666 2017-10-06 2017-10-06
2017-10-06T12:00:15 2017-10-06T12:00:15
-10017 gg 深圳 true 77 -7777 -12313342 34534 8.1
12.0 777.777 777.777 2017-10-07 2017-10-07 2017-10-07T13:20:22
2017-10-07T13:20:22
-10018 hh 杭州 false 88 6666 314234 43535356 34.124
324.0 888.888 888.888 2017-10-08 2017-10-08 2017-10-08T14:58:10
2017-10-08T14:58:10
-10019 ii 上海 true 99 -5555 1341 23434534 342.12
34234.1 999.999 999.999 2017-10-09 2017-10-09 \N \N
-10020 jj 南京 false 11 4444 -123 53623567 11.22
324.33 111.111 111.111 2017-10-10 2017-10-10 2017-10-10T16:25:42
2017-10-10T16:25:42
-10021 kk 成都 false 22 -3333 12314 674567 13.0
45464.435 222.222 222.222 2017-10-11 2017-10-11
2017-10-11T17:22:24 2017-10-11T17:22:24
-
--- !select --
-10000 aa 北京 false 11 4444 5555555 41232314 3.14
123.3423 111.111 111.111 2017-10-01 2017-10-01
2017-10-01T06:00 2017-10-01T06:00
-10001 bb 北京 false 22 3333 666 2768658 5.32
123111.3242 222.222 222.222 2017-10-02 2017-10-02
2017-10-02T07:00 2017-10-02T07:00
-10002 cc 北京 true 33 2222 453 5463456 4.321
11111.23423 333.333 333.333 2017-10-03 2017-10-03
2017-10-03T17:05:45 2017-10-03T17:05:45
-10003 dd 上海 true 44 1111 -3241 -45235 1.34
54626.324 444.444 444.444 2017-10-04 2017-10-04
2017-10-04T12:59:12 2017-10-04T12:59:12
-10004 ee 成都 false 55 -9999 21342 4513456 1.22 111.33
555.555 555.555 2017-10-05 2017-10-05 2017-10-05T11:20
2017-10-05T11:20
-10005 ff 西安 false 66 8888 64562 4356 9.133
23423.45 666.666 666.666 2017-10-06 2017-10-06
2017-10-06T12:00:15 2017-10-06T12:00:15
-10006 gg 深圳 true 77 -7777 -12313342 34534 8.1
12.0 777.777 777.777 2017-10-07 2017-10-07 2017-10-07T13:20:22
2017-10-07T13:20:22
-10007 hh 杭州 false 88 6666 314234 43535356 34.124
324.0 888.888 888.888 2017-10-08 2017-10-08 2017-10-08T14:58:10
2017-10-08T14:58:10
-10008 ii 上海 true 99 -5555 1341 23434534 342.12
34234.1 999.999 999.999 2017-10-09 2017-10-09 \N \N
-10009 jj 南京 false 11 4444 -123 53623567 11.22
324.33 111.111 111.111 2017-10-10 2017-10-10 2017-10-10T16:25:42
2017-10-10T16:25:42
-10010 kk 成都 false 22 -3333 12314 674567 13.0
45464.435 222.222 222.222 2017-10-11 2017-10-11
2017-10-11T17:22:24 2017-10-11T17:22:24
\ No newline at end of file
diff --git a/regression-test/suites/load_p0/spark_load/test_spark_load.groovy
b/regression-test/suites/load_p0/spark_load/test_spark_load.groovy
deleted file mode 100644
index 3ae313e967e..00000000000
--- a/regression-test/suites/load_p0/spark_load/test_spark_load.groovy
+++ /dev/null
@@ -1,150 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-suite("test_spark_load", "p0,external") {
- // Need spark cluster, upload data file to hdfs
- def testTable = "tbl_test_spark_load"
- def testTable2 = "tbl_test_spark_load2"
- def testResource = "spark_resource"
- def yarnAddress = "master:8032"
- def hdfsAddress = "hdfs://master:9000"
- def hdfsWorkingDir = "hdfs://master:9000/doris"
- def brokerName =getBrokerName()
- def hdfsUser = getHdfsUser()
- def hdfsPasswd = getHdfsPasswd()
-
- def create_test_table = {testTablex ->
- def result1 = sql """
- CREATE TABLE IF NOT EXISTS ${testTablex} (
- c_int int(11) NULL,
- c_char char(15) NULL,
- c_varchar varchar(100) NULL,
- c_bool boolean NULL,
- c_tinyint tinyint(4) NULL,
- c_smallint smallint(6) NULL,
- c_bigint bigint(20) NULL,
- c_largeint largeint(40) NULL,
- c_float float NULL,
- c_double double NULL,
- c_decimal decimal(6, 3) NULL,
- c_decimalv3 decimal(6, 3) NULL,
- c_date date NULL,
- c_datev2 date NULL,
- c_datetime datetime NULL,
- c_datetimev2 datetime NULL
- )
- DISTRIBUTED BY HASH(c_int) BUCKETS 1
- PROPERTIES (
- "replication_num" = "1"
- )
- """
- assertTrue(result1.size() == 1)
- assertTrue(result1[0].size() == 1)
- assertTrue(result1[0][0] == 0, "Create table should update 0 rows")
- }
-
- def create_spark_resource = {sparkType, sparkMaster, sparkQueue ->
- def result1 = sql """
- CREATE EXTERNAL RESOURCE "${testResource}"
- PROPERTIES
- (
- "type" = "spark",
- "spark.master" = "yarn",
- "spark.submit.deployMode" = "cluster",
- "spark.executor.memory" = "1g",
- "spark.yarn.queue" = "default",
- "spark.hadoop.yarn.resourcemanager.address" = "${yarnAddress}",
- "spark.hadoop.fs.defaultFS" = "${hdfsAddress}",
- "working_dir" = "${hdfsWorkingDir}",
- "broker" = "${brokerName}",
- "broker.username" = "${hdfsUser}",
- "broker.password" = "${hdfsPasswd}"
- );
- """
-
- // DDL/DML return 1 row and 3 column, the only value is update row
count
- assertTrue(result1.size() == 1)
- assertTrue(result1[0].size() == 1)
- assertTrue(result1[0][0] == 0, "Create resource should update 0 rows")
- }
-
- def load_from_hdfs_use_spark = {testTablex, testTablex2, label,
hdfsFilePath1, hdfsFilePath2 ->
- def result1= sql """
- LOAD LABEL ${label}
- (
- DATA INFILE("${hdfsFilePath1}")
- INTO TABLE ${testTablex}
- COLUMNS TERMINATED BY ",",
- DATA INFILE("${hdfsFilePath2}")
- INTO TABLE ${testTablex2}
- COLUMNS TERMINATED BY "|"
- )
- WITH RESOURCE '${testResource}'
- (
- "spark.executor.memory" = "2g",
- "spark.shuffle.compress" = "true"
- )
- PROPERTIES
- (
- "timeout" = "3600"
- );
- """
-
- assertTrue(result1.size() == 1)
- assertTrue(result1[0].size() == 1)
- assertTrue(result1[0][0] == 0, "Query OK, 0 rows affected")
- }
-
- def check_load_result = {checklabel, testTablex, testTablex2 ->
- max_try_milli_secs = 10000
- while(max_try_milli_secs) {
- result = sql "show load where label = '${checklabel}'"
- if(result[0][2] == "FINISHED") {
- sql "sync"
- qt_select "select * from ${testTablex} order by c_int"
- qt_select "select * from ${testTablex2} order by c_int"
- break
- } else {
- sleep(1000) // wait 1 second every time
- max_try_milli_secs -= 1000
- if(max_try_milli_secs <= 0) {
- assertEquals(1, 2)
- }
- }
- }
- }
-
- // if 'enableHdfs' in regression-conf.groovy has been set to true,
- if (enableHdfs()) {
- def hdfs_txt_file_path1 = uploadToHdfs
"load_p0/spark_load/all_types1.txt"
- def hdfs_txt_file_path2 = uploadToHdfs
"load_p0/spark_load/all_types2.txt"
- try {
- sql "DROP TABLE IF EXISTS ${testTable}"
- sql "DROP TABLE IF EXISTS ${testTable2}"
- create_test_table.call(testTable)
- create_test_table.call(testTable2)
- create_spark_resource.call("spark", "yarn", "default")
- def test_load_label = UUID.randomUUID().toString().replaceAll("-",
"")
- load_from_hdfs.call(testTable, testTable2, test_load_label,
hdfs_txt_file_path1, hdfs_txt_file_path2)
- check_load_result.call(test_load_label, testTable, testTable2)
-
- } finally {
- try_sql("DROP TABLE IF EXISTS ${testTable}")
- try_sql("DROP TABLE IF EXISTS ${testTable2}")
- }
- }
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
