This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new 90466cacc57 [regression](case) fix bad base (#38084)
90466cacc57 is described below
commit 90466cacc57b13abf32fda4245ebe03b7f6426c5
Author: Dongyang Li <[email protected]>
AuthorDate: Thu Jul 18 23:52:06 2024 +0800
[regression](case) fix bad base (#38084)
## Proposed changes
replace hardcode s3 properties
<!--Describe your changes.-->
---------
Co-authored-by: stephen <[email protected]>
---
.../org/apache/doris/regression/suite/Suite.groovy | 5 ++
.../test_s3_load_with_load_parallelism.groovy | 4 -
.../suites/load_p0/tvf/test_s3_tvf.groovy | 94 +++++++++++-----------
3 files changed, 53 insertions(+), 50 deletions(-)
diff --git
a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy
b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy
index 244e1c37472..a228a29d30d 100644
---
a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy
+++
b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy
@@ -752,6 +752,11 @@ class Suite implements GroovyInterceptable {
return (enableBrokerLoad != null && enableBrokerLoad.equals("true"));
}
+ String getS3Provider() {
+ String s3Provider = context.config.otherConfigs.get("s3Provider");
+ return s3Provider
+ }
+
String getS3Region() {
String s3Region = context.config.otherConfigs.get("s3Region");
return s3Region
diff --git
a/regression-test/suites/load_p0/broker_load/test_s3_load_with_load_parallelism.groovy
b/regression-test/suites/load_p0/broker_load/test_s3_load_with_load_parallelism.groovy
index 677cce88f06..353b41aefc5 100644
---
a/regression-test/suites/load_p0/broker_load/test_s3_load_with_load_parallelism.groovy
+++
b/regression-test/suites/load_p0/broker_load/test_s3_load_with_load_parallelism.groovy
@@ -80,10 +80,6 @@ suite("test_s3_load_with_load_parallelism", "load_p0") {
)
${prop}
"""
- // "AWS_ENDPOINT" = "cos.ap-beijing.myqcloud.com",
- // "AWS_ACCESS_KEY" = "AKIDd9RVMzIOI0V7Wlnbr9JG0WrhJk28zc2H",
- // "AWS_SECRET_KEY"="4uWxMhqnW3Plz97sPjqlSUXO1RhokRuO",
- // "AWS_REGION" = "ap-beijing"
def max_try_milli_secs = 600000
while (max_try_milli_secs > 0) {
diff --git a/regression-test/suites/load_p0/tvf/test_s3_tvf.groovy
b/regression-test/suites/load_p0/tvf/test_s3_tvf.groovy
index 83a93fd5d82..6bf6c8f7e73 100644
--- a/regression-test/suites/load_p0/tvf/test_s3_tvf.groovy
+++ b/regression-test/suites/load_p0/tvf/test_s3_tvf.groovy
@@ -16,7 +16,9 @@
// under the License.
suite("test_s3_tvf", "load_p0") {
-
+ def s3BucketName = getS3BucketName()
+ def s3Endpoint = getS3Endpoint()
+ def s3Region = getS3Region()
def tables = [
"agg_tbl_basic_tvf",
"dup_tbl_array_tvf",
@@ -54,7 +56,7 @@ suite("test_s3_tvf", "load_p0") {
/* normal */
for(String table : basicTables) {
attributeList.add(new TvfAttribute(table, ["K00", "K01", "K02", "K03",
"K04", "K05", "K06", "K07", "K08", "K09", "K10", "K11", "K12", "K13", "K14",
"K15", "K16", "K17", "K18"], "", "")
- .addProperty("uri",
"s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data.csv")
+ .addProperty("uri",
"s3://{$s3BucketName}.{$s3Endpoint}/regression/load/data/basic_data.csv")
.addProperty("format", "csv")
.addProperty("column_separator", "|")
.addProperty("force_parsing_by_standard_uri", "true"))
@@ -62,14 +64,14 @@ suite("test_s3_tvf", "load_p0") {
attributeList.add(new TvfAttribute("agg_tbl_basic_tvf", "c1 as k00,c2 as
k01,c3 as k02,c4 as k03,c5 as k04,c6 as k05,c7 as k06,c8 as k07,c9 as k08,c10
as k09,c11 as k10,c12 as k11,c13 as k12,c14 as k13,c15 as k14,c16 as k15,c17 as
k16,c18 as k17,c19 as k18, to_bitmap(c6) as k19, HLL_HASH(c6) as k20,
TO_QUANTILE_STATE(c5, 1.0) as k21, to_bitmap(c6) as kd19, HLL_HASH(c6) as kd20,
TO_QUANTILE_STATE(c5, 1.0) as kd21",
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18,k19,k20,k21,kd19,kd20,kd21"
,"", "")
- .addProperty("uri",
"s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data.csv")
+ .addProperty("uri",
"s3://{$s3BucketName}.{$s3Endpoint}/regression/load/data/basic_data.csv")
.addProperty("format", "csv")
.addProperty("column_separator", "|")
.addProperty("force_parsing_by_standard_uri", "true"))
for(String table : arrayTables) {
- attributeList.add(new TvfAttribute(table, ["K00", "K01", "K02", "K03",
"K04", "K05", "K06", "K07", "K08", "K09", "K10", "K11", "K12", "K13", "K14",
"K15", "K16", "K17"], "", "").addProperty("uri",
"s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_array_data.csv")
- .addProperty("uri",
"s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_array_data.csv")
+ attributeList.add(new TvfAttribute(table, ["K00", "K01", "K02", "K03",
"K04", "K05", "K06", "K07", "K08", "K09", "K10", "K11", "K12", "K13", "K14",
"K15", "K16", "K17"], "", "").addProperty("uri",
"s3://{$s3BucketName}.{$s3Endpoint}/regression/load/data/basic_array_data.csv")
+ .addProperty("uri",
"s3://{$s3BucketName}.{$s3Endpoint}/regression/load/data/basic_array_data.csv")
.addProperty("format", "csv")
.addProperty("column_separator", "|")
.addProperty("force_parsing_by_standard_uri", "true"))
@@ -77,7 +79,7 @@ suite("test_s3_tvf", "load_p0") {
for(String table : basicTables) {
attributeList.add(new TvfAttribute(table, ["k00", "k01", "k02", "k03",
"k04", "k05", "k06", "k07", "k08", "k09", "k10", "k11", "k12", "k13", "k14",
"k15", "k16", "k17", "k18"], "", "")
- .addProperty("uri",
"s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data.csv")
+ .addProperty("uri",
"s3://{$s3BucketName}.{$s3Endpoint}/regression/load/data/basic_data.csv")
.addProperty("format", "csv")
.addProperty("column_separator", "|")
.addProperty("force_parsing_by_standard_uri", "true"))
@@ -85,14 +87,14 @@ suite("test_s3_tvf", "load_p0") {
attributeList.add(new TvfAttribute("agg_tbl_basic_tvf", "c1 as k00,c2 as
k01,c3 as k02,c4 as k03,c5 as k04,c6 as k05,c7 as k06,c8 as k07,c9 as k08,c10
as k09,c11 as k10,c12 as k11,c13 as k12,c14 as k13,c15 as k14,c16 as k15,c17 as
k16,c18 as k17,c19 as k18, to_bitmap(c6) as k19, HLL_HASH(c6) as k20,
TO_QUANTILE_STATE(c5, 1.0) as k21, to_bitmap(c6) as kd19, HLL_HASH(c6) as kd20,
TO_QUANTILE_STATE(c5, 1.0) as kd21",
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18,k19,k20,k21,kd19,kd20,kd21"
,"", "")
- .addProperty("uri",
"s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data.csv")
+ .addProperty("uri",
"s3://{$s3BucketName}.{$s3Endpoint}/regression/load/data/basic_data.csv")
.addProperty("format", "csv")
.addProperty("column_separator", "|")
.addProperty("force_parsing_by_standard_uri", "true"))
for(String table : arrayTables) {
attributeList.add(new TvfAttribute(table, ["k00", "k01", "k02", "k03",
"k04", "k05", "k06", "k07", "k08", "k09", "k10", "k11", "k12", "k13", "k14",
"k15", "k16", "k17"], "", "")
- .addProperty("uri",
"s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_array_data.csv")
+ .addProperty("uri",
"s3://{$s3BucketName}.{$s3Endpoint}/regression/load/data/basic_array_data.csv")
.addProperty("format", "csv")
.addProperty("column_separator", "|")
.addProperty("force_parsing_by_standard_uri", "true"))
@@ -100,27 +102,27 @@ suite("test_s3_tvf", "load_p0") {
for(String table : basicTables) {
attributeList.add(new TvfAttribute(table, ["k00", "k01", "k02", "k03",
"k04", "k05", "k06", "k07", "k08", "k09", "k10", "k11", "k12", "k13", "k14",
"k15", "k16", "k17", "k18"], "", "")
- .addProperty("uri",
"https://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data.csv";)
+ .addProperty("uri",
"https://{$s3BucketName}.{$s3Endpoint}/regression/load/data/basic_data.csv";)
.addProperty("format", "csv")
.addProperty("column_separator", "|"))
}
attributeList.add(new TvfAttribute("agg_tbl_basic_tvf", "c1 as k00,c2 as
k01,c3 as k02,c4 as k03,c5 as k04,c6 as k05,c7 as k06,c8 as k07,c9 as k08,c10
as k09,c11 as k10,c12 as k11,c13 as k12,c14 as k13,c15 as k14,c16 as k15,c17 as
k16,c18 as k17,c19 as k18, to_bitmap(c6) as k19, HLL_HASH(c6) as k20,
TO_QUANTILE_STATE(c5, 1.0) as k21, to_bitmap(c6) as kd19, HLL_HASH(c6) as kd20,
TO_QUANTILE_STATE(c5, 1.0) as kd21",
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18,k19,k20,k21,kd19,kd20,kd21"
,"", "")
- .addProperty("uri",
"https://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data.csv";)
+ .addProperty("uri",
"https://{$s3BucketName}.{$s3Endpoint}/regression/load/data/basic_data.csv";)
.addProperty("format", "csv")
.addProperty("column_separator", "|"))
for(String table : arrayTables) {
attributeList.add(new TvfAttribute(table, ["k00", "k01", "k02", "k03",
"k04", "k05", "k06", "k07", "k08", "k09", "k10", "k11", "k12", "k13", "k14",
"k15", "k16", "k17"], "", "")
- .addProperty("uri",
"https://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_array_data.csv";)
+ .addProperty("uri",
"https://{$s3BucketName}.{$s3Endpoint}/regression/load/data/basic_array_data.csv";)
.addProperty("format", "csv")
.addProperty("column_separator", "|"))
}
// for(String table : basicTables) {
// attributeList.add(new TvfAttribute(table, ["k00", "k01", "k02",
"k03", "k04", "k05", "k06", "k07", "k08", "k09", "k10", "k11", "k12", "k13",
"k14", "k15", "k16", "k17", "k18"], "", "")
-// .addProperty("uri",
"https://cos.ap-beijing.myqcloud.com/doris-build-1308700295/regression/load/data/basic_data.csv";)
+// .addProperty("uri",
"https://{$s3Endpoint}/{$s3BucketName}/regression/load/data/basic_data.csv";)
// .addProperty("format", "csv")
// .addProperty("column_separator", "|")
// .addProperty("use_path_style", "true"))
@@ -128,14 +130,14 @@ suite("test_s3_tvf", "load_p0") {
//
// attributeList.add(new TvfAttribute("agg_tbl_basic_tvf", "c1 as k00,c2 as
k01,c3 as k02,c4 as k03,c5 as k04,c6 as k05,c7 as k06,c8 as k07,c9 as k08,c10
as k09,c11 as k10,c12 as k11,c13 as k12,c14 as k13,c15 as k14,c16 as k15,c17 as
k16,c18 as k17,c19 as k18, to_bitmap(c6) as k19, HLL_HASH(c6) as k20,
TO_QUANTILE_STATE(c5, 1.0) as k21, to_bitmap(c6) as kd19, HLL_HASH(c6) as kd20,
TO_QUANTILE_STATE(c5, 1.0) as kd21",
//
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18,k19,k20,k21,kd19,kd20,kd21"
,"", "")
-// .addProperty("uri",
"https://cos.ap-beijing.myqcloud.com/doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data.csv";)
+// .addProperty("uri",
"https://{$s3Endpoint}/{$s3BucketName}.{$s3Endpoint}/regression/load/data/basic_data.csv";)
// .addProperty("format", "csv")
// .addProperty("column_separator", "|")
// .addProperty("use_path_style", "true"))
//
// for(String table : arrayTables) {
// attributeList.add(new TvfAttribute(table, ["k00", "k01", "k02",
"k03", "k04", "k05", "k06", "k07", "k08", "k09", "k10", "k11", "k12", "k13",
"k14", "k15", "k16", "k17"], "", "")
-// .addProperty("uri",
"https://cos.ap-beijing.myqcloud.com/doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_array_data.csv";)
+// .addProperty("uri",
"https://{$s3Endpoint}/{$s3BucketName}.{$s3Endpoint}/regression/load/data/basic_array_data.csv";)
// .addProperty("format", "csv")
// .addProperty("column_separator", "|")
// .addProperty("use_path_style", "true"))
@@ -144,7 +146,7 @@ suite("test_s3_tvf", "load_p0") {
/* error */
for(String table : basicTables) {
attributeList.add(new TvfAttribute(table, ["k00", "k01", "k02", "k03",
"k04", "k05", "k06", "k07", "k08", "k09", "k10", "k11", "k12", "k13", "k14",
"k15", "k16", "k17", "k18"], "", "", true)
- .addProperty("uri",
"s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data_with_errors.csv")
+ .addProperty("uri",
"s3://{$s3BucketName}.{$s3Endpoint}/regression/load/data/basic_data_with_errors.csv")
.addProperty("format", "csv")
.addProperty("column_separator", "|")
.addProperty("force_parsing_by_standard_uri", "true"))
@@ -152,14 +154,14 @@ suite("test_s3_tvf", "load_p0") {
attributeList.add(new TvfAttribute("agg_tbl_basic_tvf", "c1 as k00,c2 as
k01,c3 as k02,c4 as k03,c5 as k04,c6 as k05,c7 as k06,c8 as k07,c9 as k08,c10
as k09,c11 as k10,c12 as k11,c13 as k12,c14 as k13,c15 as k14,c16 as k15,c17 as
k16,c18 as k17,c19 as k18, to_bitmap(c6) as k19, HLL_HASH(c6) as k20,
TO_QUANTILE_STATE(c5, 1.0) as k21, to_bitmap(c6) as kd19, HLL_HASH(c6) as kd20,
TO_QUANTILE_STATE(c5, 1.0) as kd21",
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18,k19,k20,k21,kd19,kd20,kd21"
,"", "", true)
- .addProperty("uri",
"s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data_with_errors.csv")
+ .addProperty("uri",
"s3://{$s3BucketName}.{$s3Endpoint}/regression/load/data/basic_data_with_errors.csv")
.addProperty("format", "csv")
.addProperty("column_separator", "|")
.addProperty("force_parsing_by_standard_uri", "true"))
for(String table : arrayTables) {
attributeList.add(new TvfAttribute(table, ["k00", "k01", "k02", "k03",
"k04", "k05", "k06", "k07", "k08", "k09", "k10", "k11", "k12", "k13", "k14",
"k15", "k16", "k17", "kd01", "kd02", "kd03", "kd04", "kd05", "kd06", "kd07",
"kd08", "kd09", "kd10", "kd11", "kd12", "kd13", "kd14", "kd15", "kd16"], "",
"", true)
- .addProperty("uri",
"s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_array_data_with_errors.csv")
+ .addProperty("uri",
"s3://{$s3BucketName}.{$s3Endpoint}/regression/load/data/basic_array_data_with_errors.csv")
.addProperty("format", "csv")
.addProperty("column_separator", "|")
.addProperty("force_parsing_by_standard_uri", "true"))
@@ -168,7 +170,7 @@ suite("test_s3_tvf", "load_p0") {
/* skip lines */
for(String table : basicTables) {
attributeList.add(new TvfAttribute(table, ["k00", "k01", "k02", "k03",
"k04", "k05", "k06", "k07", "k08", "k09", "k10", "k11", "k12", "k13", "k14",
"k15", "k16", "k17", "k18"], "", "")
- .addProperty("uri",
"s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data_with_errors.csv")
+ .addProperty("uri",
"s3://{$s3BucketName}.{$s3Endpoint}/regression/load/data/basic_data_with_errors.csv")
.addProperty("format", "csv")
.addProperty("column_separator", "|")
.addProperty("skip_lines", "10")
@@ -177,7 +179,7 @@ suite("test_s3_tvf", "load_p0") {
attributeList.add(new TvfAttribute("agg_tbl_basic_tvf", "c1 as k00,c2 as
k01,c3 as k02,c4 as k03,c5 as k04,c6 as k05,c7 as k06,c8 as k07,c9 as k08,c10
as k09,c11 as k10,c12 as k11,c13 as k12,c14 as k13,c15 as k14,c16 as k15,c17 as
k16,c18 as k17,c19 as k18, to_bitmap(c6) as k19, HLL_HASH(c6) as k20,
TO_QUANTILE_STATE(c5, 1.0) as k21, to_bitmap(c6) as kd19, HLL_HASH(c6) as kd20,
TO_QUANTILE_STATE(c5, 1.0) as kd21",
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18,k19,k20,k21,kd19,kd20,kd21"
,"", "")
- .addProperty("uri",
"s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data_with_errors.csv")
+ .addProperty("uri",
"s3://{$s3BucketName}.{$s3Endpoint}/regression/load/data/basic_data_with_errors.csv")
.addProperty("format", "csv")
.addProperty("column_separator", "|")
.addProperty("skip_lines", "10")
@@ -185,7 +187,7 @@ suite("test_s3_tvf", "load_p0") {
for(String table : arrayTables) {
attributeList.add(new TvfAttribute(table, ["k00", "k01", "k02", "k03",
"k04", "k05", "k06", "k07", "k08", "k09", "k10", "k11", "k12", "k13", "k14",
"k15", "k16", "k17"], "", "")
- .addProperty("uri",
"s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_array_data_with_errors.csv")
+ .addProperty("uri",
"s3://{$s3BucketName}.{$s3Endpoint}/regression/load/data/basic_array_data_with_errors.csv")
.addProperty("format", "csv")
.addProperty("column_separator", "|")
.addProperty("skip_lines", "10")
@@ -195,21 +197,21 @@ suite("test_s3_tvf", "load_p0") {
/* compress type */
for(String table : basicTables) {
attributeList.add(new TvfAttribute(table, ["k00", "k01", "k02", "k03",
"k04", "k05", "k06", "k07", "k08", "k09", "k10", "k11", "k12", "k13", "k14",
"k15", "k16", "k17", "k18"], "", "")
- .addProperty("uri",
"https://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data.csv.gz";)
+ .addProperty("uri",
"https://{$s3BucketName}.{$s3Endpoint}/regression/load/data/basic_data.csv.gz";)
.addProperty("format", "csv")
.addProperty("column_separator",
"|").addProperty("compress_type", "GZ"))
}
attributeList.add(new TvfAttribute("agg_tbl_basic_tvf", "c1 as k00,c2 as
k01,c3 as k02,c4 as k03,c5 as k04,c6 as k05,c7 as k06,c8 as k07,c9 as k08,c10
as k09,c11 as k10,c12 as k11,c13 as k12,c14 as k13,c15 as k14,c16 as k15,c17 as
k16,c18 as k17,c19 as k18, to_bitmap(c6) as k19, HLL_HASH(c6) as k20,
TO_QUANTILE_STATE(c5, 1.0) as k21, to_bitmap(c6) as kd19, HLL_HASH(c6) as kd20,
TO_QUANTILE_STATE(c5, 1.0) as kd21",
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18,k19,k20,k21,kd19,kd20,kd21"
,"", "")
- .addProperty("uri",
"https://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data.csv.gz";)
+ .addProperty("uri",
"https://{$s3BucketName}.{$s3Endpoint}/regression/load/data/basic_data.csv.gz";)
.addProperty("format", "csv")
.addProperty("column_separator", "|")
.addProperty("compress_type", "GZ"))
for(String table : arrayTables) {
attributeList.add(new TvfAttribute(table, ["k00", "k01", "k02", "k03",
"k04", "k05", "k06", "k07", "k08", "k09", "k10", "k11", "k12", "k13", "k14",
"k15", "k16", "k17"], "", "")
- .addProperty("uri",
"https://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_array_data.csv.gz";)
+ .addProperty("uri",
"https://{$s3BucketName}.{$s3Endpoint}/regression/load/data/basic_array_data.csv.gz";)
.addProperty("format", "csv")
.addProperty("column_separator", "|")
.addProperty("compress_type", "GZ"))
@@ -217,21 +219,21 @@ suite("test_s3_tvf", "load_p0") {
for(String table : basicTables) {
attributeList.add(new TvfAttribute(table, ["k00", "k01", "k02", "k03",
"k04", "k05", "k06", "k07", "k08", "k09", "k10", "k11", "k12", "k13", "k14",
"k15", "k16", "k17", "k18"], "", "")
- .addProperty("uri",
"https://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data.csv.bz2";)
+ .addProperty("uri",
"https://{$s3BucketName}.{$s3Endpoint}/regression/load/data/basic_data.csv.bz2";)
.addProperty("format", "csv")
.addProperty("column_separator",
"|").addProperty("compress_type", "BZ2"))
}
attributeList.add(new TvfAttribute("agg_tbl_basic_tvf", "c1 as k00,c2 as
k01,c3 as k02,c4 as k03,c5 as k04,c6 as k05,c7 as k06,c8 as k07,c9 as k08,c10
as k09,c11 as k10,c12 as k11,c13 as k12,c14 as k13,c15 as k14,c16 as k15,c17 as
k16,c18 as k17,c19 as k18, to_bitmap(c6) as k19, HLL_HASH(c6) as k20,
TO_QUANTILE_STATE(c5, 1.0) as k21, to_bitmap(c6) as kd19, HLL_HASH(c6) as kd20,
TO_QUANTILE_STATE(c5, 1.0) as kd21",
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18,k19,k20,k21,kd19,kd20,kd21"
,"", "")
- .addProperty("uri",
"https://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data.csv.bz2";)
+ .addProperty("uri",
"https://{$s3BucketName}.{$s3Endpoint}/regression/load/data/basic_data.csv.bz2";)
.addProperty("format", "csv")
.addProperty("column_separator", "|")
.addProperty("compress_type", "BZ2"))
for(String table : arrayTables) {
attributeList.add(new TvfAttribute(table, ["k00", "k01", "k02", "k03",
"k04", "k05", "k06", "k07", "k08", "k09", "k10", "k11", "k12", "k13", "k14",
"k15", "k16", "k17"], "", "")
- .addProperty("uri",
"https://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_array_data.csv.bz2";)
+ .addProperty("uri",
"https://{$s3BucketName}.{$s3Endpoint}/regression/load/data/basic_array_data.csv.bz2";)
.addProperty("format", "csv")
.addProperty("column_separator", "|")
.addProperty("compress_type", "BZ2"))
@@ -239,21 +241,21 @@ suite("test_s3_tvf", "load_p0") {
for(String table : basicTables) {
attributeList.add(new TvfAttribute(table, ["k00", "k01", "k02", "k03",
"k04", "k05", "k06", "k07", "k08", "k09", "k10", "k11", "k12", "k13", "k14",
"k15", "k16", "k17", "k18"], "", "")
- .addProperty("uri",
"https://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data.csv.lz4";)
+ .addProperty("uri",
"https://{$s3BucketName}.{$s3Endpoint}/regression/load/data/basic_data.csv.lz4";)
.addProperty("format", "csv")
.addProperty("column_separator",
"|").addProperty("compress_type", "LZ4FRAME"))
}
attributeList.add(new TvfAttribute("agg_tbl_basic_tvf", "c1 as k00,c2 as
k01,c3 as k02,c4 as k03,c5 as k04,c6 as k05,c7 as k06,c8 as k07,c9 as k08,c10
as k09,c11 as k10,c12 as k11,c13 as k12,c14 as k13,c15 as k14,c16 as k15,c17 as
k16,c18 as k17,c19 as k18, to_bitmap(c6) as k19, HLL_HASH(c6) as k20,
TO_QUANTILE_STATE(c5, 1.0) as k21, to_bitmap(c6) as kd19, HLL_HASH(c6) as kd20,
TO_QUANTILE_STATE(c5, 1.0) as kd21",
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18,k19,k20,k21,kd19,kd20,kd21"
,"", "")
- .addProperty("uri",
"https://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data.csv.lz4";)
+ .addProperty("uri",
"https://{$s3BucketName}.{$s3Endpoint}/regression/load/data/basic_data.csv.lz4";)
.addProperty("format", "csv")
.addProperty("column_separator", "|")
.addProperty("compress_type", "LZ4FRAME"))
for(String table : arrayTables) {
attributeList.add(new TvfAttribute(table, ["k00", "k01", "k02", "k03",
"k04", "k05", "k06", "k07", "k08", "k09", "k10", "k11", "k12", "k13", "k14",
"k15", "k16", "k17"], "", "")
- .addProperty("uri",
"https://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_array_data.csv.lz4";)
+ .addProperty("uri",
"https://{$s3BucketName}.{$s3Endpoint}/regression/load/data/basic_array_data.csv.lz4";)
.addProperty("format", "csv")
.addProperty("column_separator", "|")
.addProperty("compress_type", "LZ4FRAME"))
@@ -261,7 +263,7 @@ suite("test_s3_tvf", "load_p0") {
for(String table : basicTables) {
attributeList.add(new TvfAttribute(table, ["k00", "k01", "k02", "k03",
"k04", "k05", "k06", "k07", "k08", "k09", "k10", "k11", "k12", "k13", "k14",
"k15", "k16", "k17", "k18"], "WHERE c1 > 50", "")
- .addProperty("uri",
"s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data.csv")
+ .addProperty("uri",
"s3://{$s3BucketName}.{$s3Endpoint}/regression/load/data/basic_data.csv")
.addProperty("format", "csv")
.addProperty("column_separator", "|")
.addProperty("force_parsing_by_standard_uri", "true"))
@@ -269,14 +271,14 @@ suite("test_s3_tvf", "load_p0") {
attributeList.add(new TvfAttribute("agg_tbl_basic_tvf", "c1 as k00,c2 as
k01,c3 as k02,c4 as k03,c5 as k04,c6 as k05,c7 as k06,c8 as k07,c9 as k08,c10
as k09,c11 as k10,c12 as k11,c13 as k12,c14 as k13,c15 as k14,c16 as k15,c17 as
k16,c18 as k17,c19 as k18, to_bitmap(c6) as k19, HLL_HASH(c6) as k20,
TO_QUANTILE_STATE(c5, 1.0) as k21, to_bitmap(c6) as kd19, HLL_HASH(c6) as kd20,
TO_QUANTILE_STATE(c5, 1.0) as kd21",
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18,k19,k20,k21,kd19,kd20,kd21"
,"WHERE c1 > 50", "")
- .addProperty("uri",
"s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data.csv")
+ .addProperty("uri",
"s3://{$s3BucketName}.{$s3Endpoint}/regression/load/data/basic_data.csv")
.addProperty("format", "csv")
.addProperty("column_separator", "|")
.addProperty("force_parsing_by_standard_uri", "true"))
for(String table : arrayTables) {
attributeList.add(new TvfAttribute(table, ["k00", "k01", "k02", "k03",
"k04", "k05", "k06", "k07", "k08", "k09", "k10", "k11", "k12", "k13", "k14",
"k15", "k16", "k17"], "WHERE c1 > 50", "")
- .addProperty("uri",
"s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_array_data.csv")
+ .addProperty("uri",
"s3://{$s3BucketName}.{$s3Endpoint}/regression/load/data/basic_array_data.csv")
.addProperty("format", "csv")
.addProperty("column_separator", "|")
.addProperty("force_parsing_by_standard_uri", "true"))
@@ -284,7 +286,7 @@ suite("test_s3_tvf", "load_p0") {
for(String table : uniqTable) {
attributeList.add(new TvfAttribute(table, ["k00", "k01", "k02", "k03",
"k04", "k05", "k06", "k07", "k08", "k09", "k10", "k11", "k12", "k13", "k14",
"k15", "k16", "k17", "k18"], "", "ORDER BY c1")
- .addProperty("uri",
"s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data.csv")
+ .addProperty("uri",
"s3://{$s3BucketName}.{$s3Endpoint}/regression/load/data/basic_data.csv")
.addProperty("format", "csv")
.addProperty("column_separator", "|")
.addProperty("force_parsing_by_standard_uri", "true"))
@@ -292,7 +294,7 @@ suite("test_s3_tvf", "load_p0") {
for(String table : basicTables) {
attributeList.add(new TvfAttribute(table,
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18","k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18",
"", "")
- .addProperty("uri",
"s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data.parq")
+ .addProperty("uri",
"s3://{$s3BucketName}.{$s3Endpoint}/regression/load/data/basic_data.parq")
.addProperty("format", "parquet")
.addProperty("column_separator", "|")
.addProperty("force_parsing_by_standard_uri", "true"))
@@ -300,14 +302,14 @@ suite("test_s3_tvf", "load_p0") {
attributeList.add(new TvfAttribute("agg_tbl_basic_tvf",
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18,
to_bitmap(k05) as k19, HLL_HASH(k05) as k20, TO_QUANTILE_STATE(k04, 1.0) as
k21, to_bitmap(k05) as kd19, HLL_HASH(k05) as kd20, TO_QUANTILE_STATE(k04, 1.0)
as kd21",
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18,k19,k20,k21,kd19,kd20,kd21"
,"", "")
- .addProperty("uri",
"s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data.parq")
+ .addProperty("uri",
"s3://{$s3BucketName}.{$s3Endpoint}/regression/load/data/basic_data.parq")
.addProperty("format", "parquet")
.addProperty("column_separator", "|")
.addProperty("force_parsing_by_standard_uri", "true"))
for(String table : arrayTables) {
attributeList.add(new TvfAttribute(table,
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17",
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17", "",
"")
- .addProperty("uri",
"s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_array_data.parq")
+ .addProperty("uri",
"s3://{$s3BucketName}.{$s3Endpoint}/regression/load/data/basic_array_data.parq")
.addProperty("format", "parquet")
.addProperty("column_separator", "|")
.addProperty("force_parsing_by_standard_uri", "true"))
@@ -315,7 +317,7 @@ suite("test_s3_tvf", "load_p0") {
for(String table : basicTables) {
attributeList.add(new TvfAttribute(table,
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18","k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18",
"", "")
- .addProperty("uri",
"s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data.orc")
+ .addProperty("uri",
"s3://{$s3BucketName}.{$s3Endpoint}/regression/load/data/basic_data.orc")
.addProperty("format", "orc")
.addProperty("column_separator", "|")
.addProperty("force_parsing_by_standard_uri", "true"))
@@ -323,14 +325,14 @@ suite("test_s3_tvf", "load_p0") {
attributeList.add(new TvfAttribute("agg_tbl_basic_tvf",
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18,
to_bitmap(k05) as k19, HLL_HASH(k05) as k20, TO_QUANTILE_STATE(k04, 1.0) as
k21, to_bitmap(k05) as kd19, HLL_HASH(k05) as kd20, TO_QUANTILE_STATE(k04, 1.0)
as kd21",
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18,k19,k20,k21,kd19,kd20,kd21"
,"", "")
- .addProperty("uri",
"s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data.orc")
+ .addProperty("uri",
"s3://{$s3BucketName}.{$s3Endpoint}/regression/load/data/basic_data.orc")
.addProperty("format", "orc")
.addProperty("column_separator", "|")
.addProperty("force_parsing_by_standard_uri", "true"))
for(String table : arrayTables) {
attributeList.add(new TvfAttribute(table,
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17",
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17", "",
"")
- .addProperty("uri",
"s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_array_data.orc")
+ .addProperty("uri",
"s3://{$s3BucketName}.{$s3Endpoint}/regression/load/data/basic_array_data.orc")
.addProperty("format", "orc")
.addProperty("column_separator", "|")
.addProperty("force_parsing_by_standard_uri", "true"))
@@ -338,7 +340,7 @@ suite("test_s3_tvf", "load_p0") {
for(String table : basicTables) {
attributeList.add(new TvfAttribute(table,
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18","k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18",
"", "")
- .addProperty("uri",
"s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data.json")
+ .addProperty("uri",
"s3://{$s3BucketName}.{$s3Endpoint}/regression/load/data/basic_data.json")
.addProperty("format", "json")
.addProperty("read_json_by_line", "false")
.addProperty("strip_outer_array", "true")
@@ -348,7 +350,7 @@ suite("test_s3_tvf", "load_p0") {
attributeList.add(new TvfAttribute("agg_tbl_basic_tvf",
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18,
to_bitmap(k05) as k19, HLL_HASH(k05) as k20, TO_QUANTILE_STATE(k04, 1.0) as
k21, to_bitmap(k05) as kd19, HLL_HASH(k05) as kd20, TO_QUANTILE_STATE(k04, 1.0)
as kd21",
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18,k19,k20,k21,kd19,kd20,kd21"
,"", "")
- .addProperty("uri",
"s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data.json")
+ .addProperty("uri",
"s3://{$s3BucketName}.{$s3Endpoint}/regression/load/data/basic_data.json")
.addProperty("format", "json")
.addProperty("read_json_by_line", "false")
.addProperty("strip_outer_array", "true")
@@ -357,7 +359,7 @@ suite("test_s3_tvf", "load_p0") {
for(String table : arrayTables) {
attributeList.add(new TvfAttribute(table,
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17",
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17", "",
"")
- .addProperty("uri",
"s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_array_data.json")
+ .addProperty("uri",
"s3://{$s3BucketName}.{$s3Endpoint}/regression/load/data/basic_array_data.json")
.addProperty("format", "json")
.addProperty("read_json_by_line", "false")
.addProperty("strip_outer_array", "true")
@@ -367,7 +369,7 @@ suite("test_s3_tvf", "load_p0") {
for(String table : basicTables) {
attributeList.add(new TvfAttribute(table,
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18","k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18",
"", "")
- .addProperty("uri",
"s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data_by_line.json")
+ .addProperty("uri",
"s3://{$s3BucketName}.{$s3Endpoint}/regression/load/data/basic_data_by_line.json")
.addProperty("format", "json")
.addProperty("read_json_by_line", "true")
.addProperty("strip_outer_array", "false")
@@ -377,7 +379,7 @@ suite("test_s3_tvf", "load_p0") {
attributeList.add(new TvfAttribute("agg_tbl_basic_tvf",
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18,
to_bitmap(k05) as k19, HLL_HASH(k05) as k20, TO_QUANTILE_STATE(k04, 1.0) as
k21, to_bitmap(k05) as kd19, HLL_HASH(k05) as kd20, TO_QUANTILE_STATE(k04, 1.0)
as kd21",
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18,k19,k20,k21,kd19,kd20,kd21"
,"", "")
- .addProperty("uri",
"s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data_by_line.json")
+ .addProperty("uri",
"s3://{$s3BucketName}.{$s3Endpoint}/regression/load/data/basic_data_by_line.json")
.addProperty("format", "json")
.addProperty("read_json_by_line", "true")
.addProperty("strip_outer_array", "false")
@@ -386,7 +388,7 @@ suite("test_s3_tvf", "load_p0") {
for(String table : arrayTables) {
attributeList.add(new TvfAttribute(table,
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17",
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17", "",
"")
- .addProperty("uri",
"s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_array_data_by_line.json")
+ .addProperty("uri",
"s3://{$s3BucketName}.{$s3Endpoint}/regression/load/data/basic_array_data_by_line.json")
.addProperty("format", "json")
.addProperty("read_json_by_line", "true")
.addProperty("strip_outer_array", "false")
@@ -415,7 +417,7 @@ suite("test_s3_tvf", "load_p0") {
FROM S3 (
"s3.access_key" = "$ak",
"s3.secret_key" = "$sk",
- "s3.region" = "ap-beijing",
+ "s3.region" = "${s3Region}",
${prop}
) ${attribute.whereClause}
${attribute.orderByClause}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
