This is an automated email from the ASF dual-hosted git repository.
hellostephen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 9c4bbcb0040 [chore](tool) fix stream load header (#51554)
9c4bbcb0040 is described below
commit 9c4bbcb00402a7e1b9aa0652534a7c2eda683678
Author: feifeifeimoon <[email protected]>
AuthorDate: Fri Jun 6 15:18:32 2025 +0800
[chore](tool) fix stream load header (#51554)
### What problem does this PR solve?
---
tools/clickbench-tools/load-clickbench-data.sh | 2 ++
tools/ssb-tools/bin/load-ssb-data.sh | 20 ++++++++--------
tools/tpcds-tools/bin/load-tpcds-data.sh | 2 ++
tools/tpch-tools/bin/load-tpch-data.sh | 32 +++++++++++++-------------
4 files changed, 30 insertions(+), 26 deletions(-)
diff --git a/tools/clickbench-tools/load-clickbench-data.sh
b/tools/clickbench-tools/load-clickbench-data.sh
index 208bdaff52d..4f08671a812 100755
--- a/tools/clickbench-tools/load-clickbench-data.sh
+++ b/tools/clickbench-tools/load-clickbench-data.sh
@@ -146,12 +146,14 @@ function load() {
curl --location-trusted \
-u $USER:$PASSWORD \
-T "$DATA_DIR/hits_split${i}" \
+ -H "Expect: 100-continue" \
-H
"columns:WatchID,JavaEnable,Title,GoodEvent,EventTime,EventDate,CounterID,ClientIP,RegionID,UserID,CounterClass,OS,UserAgent,URL,Referer,IsRefresh,RefererCategoryID,RefererRegionID,URLCategoryID,URLRegionID,ResolutionWidth,ResolutionHeight,ResolutionDepth,FlashMajor,FlashMinor,FlashMinor2,NetMajor,NetMinor,UserAgentMajor,UserAgentMinor,CookieEnable,JavascriptEnable,IsMobile,MobilePhone,MobilePhoneModel,Params,IPNetworkID,TraficSourceID,SearchEngineID,SearchPhrase,AdvEn
[...]
http://$FE_HOST:$FE_HTTP_PORT/api/$DB/hits/_stream_load
else
curl --location-trusted \
-u $USER:$PASSWORD \
-T "$DATA_DIR/hits_split${i}" \
+ -H "Expect: 100-continue" \
-H "label:${TXN_ID}_${i}" \
-H
"columns:WatchID,JavaEnable,Title,GoodEvent,EventTime,EventDate,CounterID,ClientIP,RegionID,UserID,CounterClass,OS,UserAgent,URL,Referer,IsRefresh,RefererCategoryID,RefererRegionID,URLCategoryID,URLRegionID,ResolutionWidth,ResolutionHeight,ResolutionDepth,FlashMajor,FlashMinor,FlashMinor2,NetMajor,NetMinor,UserAgentMajor,UserAgentMinor,CookieEnable,JavascriptEnable,IsMobile,MobilePhone,MobilePhoneModel,Params,IPNetworkID,TraficSourceID,SearchEngineID,SearchPhrase,AdvEn
[...]
http://$FE_HOST:$FE_HTTP_PORT/api/$DB/hits/_stream_load
diff --git a/tools/ssb-tools/bin/load-ssb-data.sh
b/tools/ssb-tools/bin/load-ssb-data.sh
index c96f7250b61..d45bb176b5c 100755
--- a/tools/ssb-tools/bin/load-ssb-data.sh
+++ b/tools/ssb-tools/bin/load-ssb-data.sh
@@ -286,12 +286,12 @@ echo "==========Start to load data into ssb
tables=========="
echo 'Loading data for table: part'
if [[ -z ${TXN_ID} ]]; then
curl --location-trusted -u "${USER}":"${PASSWORD}" \
- -H "column_separator:|" \
+ -H "Expect: 100-continue" -H "column_separator:|" \
-H
"columns:p_partkey,p_name,p_mfgr,p_category,p_brand,p_color,p_type,p_size,p_container,p_dummy"
\
-T "${SSB_DATA_DIR}"/part.tbl
http://"${FE_HOST}":"${FE_HTTP_PORT}"/api/"${DB}"/part/_stream_load
else
curl --location-trusted -u "${USER}":"${PASSWORD}" \
- -H "label:${TXN_ID}_part" -H "column_separator:|" \
+ -H "Expect: 100-continue" -H "label:${TXN_ID}_part" -H
"column_separator:|" \
-H
"columns:p_partkey,p_name,p_mfgr,p_category,p_brand,p_color,p_type,p_size,p_container,p_dummy"
\
-T "${SSB_DATA_DIR}"/part.tbl
http://"${FE_HOST}":"${FE_HTTP_PORT}"/api/"${DB}"/part/_stream_load
fi
@@ -299,12 +299,12 @@ fi
echo 'Loading data for table: dates'
if [[ -z ${TXN_ID} ]]; then
curl --location-trusted -u "${USER}":"${PASSWORD}" \
- -H "column_separator:|" \
+ -H "Expect: 100-continue" -H "column_separator:|" \
-H
"columns:d_datekey,d_date,d_dayofweek,d_month,d_year,d_yearmonthnum,d_yearmonth,d_daynuminweek,d_daynuminmonth,d_daynuminyear,d_monthnuminyear,d_weeknuminyear,d_sellingseason,d_lastdayinweekfl,d_lastdayinmonthfl,d_holidayfl,d_weekdayfl,d_dummy"
\
-T "${SSB_DATA_DIR}"/date.tbl
http://"${FE_HOST}":"${FE_HTTP_PORT}"/api/"${DB}"/dates/_stream_load
else
curl --location-trusted -u "${USER}":"${PASSWORD}" \
- -H "label:${TXN_ID}_date" -H "column_separator:|" \
+ -H "Expect: 100-continue" -H "label:${TXN_ID}_date" -H
"column_separator:|" \
-H
"columns:d_datekey,d_date,d_dayofweek,d_month,d_year,d_yearmonthnum,d_yearmonth,d_daynuminweek,d_daynuminmonth,d_daynuminyear,d_monthnuminyear,d_weeknuminyear,d_sellingseason,d_lastdayinweekfl,d_lastdayinmonthfl,d_holidayfl,d_weekdayfl,d_dummy"
\
-T "${SSB_DATA_DIR}"/date.tbl
http://"${FE_HOST}":"${FE_HTTP_PORT}"/api/"${DB}"/dates/_stream_load
fi
@@ -312,12 +312,12 @@ fi
echo 'Loading data for table: supplier'
if [[ -z ${TXN_ID} ]]; then
curl --location-trusted -u "${USER}":"${PASSWORD}" \
- -H "column_separator:|" \
+ -H "Expect: 100-continue" -H "column_separator:|" \
-H
"columns:s_suppkey,s_name,s_address,s_city,s_nation,s_region,s_phone,s_dummy" \
-T "${SSB_DATA_DIR}"/supplier.tbl
http://"${FE_HOST}":"${FE_HTTP_PORT}"/api/"${DB}"/supplier/_stream_load
else
curl --location-trusted -u "${USER}":"${PASSWORD}" \
- -H "label:${TXN_ID}_supplier" -H "column_separator:|" \
+ -H "Expect: 100-continue" -H "label:${TXN_ID}_supplier" -H
"column_separator:|" \
-H
"columns:s_suppkey,s_name,s_address,s_city,s_nation,s_region,s_phone,s_dummy" \
-T "${SSB_DATA_DIR}"/supplier.tbl
http://"${FE_HOST}":"${FE_HTTP_PORT}"/api/"${DB}"/supplier/_stream_load
fi
@@ -325,12 +325,12 @@ fi
echo 'Loading data for table: customer'
if [[ -z ${TXN_ID} ]]; then
curl --location-trusted -u "${USER}":"${PASSWORD}" \
- -H "column_separator:|" \
+ -H "Expect: 100-continue" -H "column_separator:|" \
-H
"columns:c_custkey,c_name,c_address,c_city,c_nation,c_region,c_phone,c_mktsegment,no_use"
\
-T "${SSB_DATA_DIR}"/customer.tbl
http://"${FE_HOST}":"${FE_HTTP_PORT}"/api/"${DB}"/customer/_stream_load
else
curl --location-trusted -u "${USER}":"${PASSWORD}" \
- -H "label:${TXN_ID}_customer" -H "column_separator:|" \
+ -H "Expect: 100-continue" -H "label:${TXN_ID}_customer" -H
"column_separator:|" \
-H
"columns:c_custkey,c_name,c_address,c_city,c_nation,c_region,c_phone,c_mktsegment,no_use"
\
-T "${SSB_DATA_DIR}"/customer.tbl
http://"${FE_HOST}":"${FE_HTTP_PORT}"/api/"${DB}"/customer/_stream_load
fi
@@ -342,12 +342,12 @@ function load() {
local FILE_ID="${@//*./}"
if [[ -z ${TXN_ID} ]]; then
curl --location-trusted -u "${USER}":"${PASSWORD}" \
- -H "column_separator:|" \
+ -H "Expect: 100-continue" -H "column_separator:|" \
-H
"columns:lo_orderkey,lo_linenumber,lo_custkey,lo_partkey,lo_suppkey,lo_orderdate,lo_orderpriority,lo_shippriority,lo_quantity,lo_extendedprice,lo_ordtotalprice,lo_discount,lo_revenue,lo_supplycost,lo_tax,lo_commitdate,lo_shipmode,lo_dummy"
\
-T "$@"
http://"${FE_HOST}":"${FE_HTTP_PORT}"/api/"${DB}"/lineorder/_stream_load
else
curl --location-trusted -u "${USER}":"${PASSWORD}" \
- -H "label:${TXN_ID}_lineorder_${FILE_ID}" -H "column_separator:|" \
+ -H "Expect: 100-continue" -H
"label:${TXN_ID}_lineorder_${FILE_ID}" -H "column_separator:|" \
-H
"columns:lo_orderkey,lo_linenumber,lo_custkey,lo_partkey,lo_suppkey,lo_orderdate,lo_orderpriority,lo_shippriority,lo_quantity,lo_extendedprice,lo_ordtotalprice,lo_discount,lo_revenue,lo_supplycost,lo_tax,lo_commitdate,lo_shipmode,lo_dummy"
\
-T "$@"
http://"${FE_HOST}":"${FE_HTTP_PORT}"/api/"${DB}"/lineorder/_stream_load
fi
diff --git a/tools/tpcds-tools/bin/load-tpcds-data.sh
b/tools/tpcds-tools/bin/load-tpcds-data.sh
index 95b6ebbbc1b..6197d0d6d18 100755
--- a/tools/tpcds-tools/bin/load-tpcds-data.sh
+++ b/tools/tpcds-tools/bin/load-tpcds-data.sh
@@ -182,6 +182,7 @@ for table_name in ${!table_columns[*]}; do
ret=$(curl \
--location-trusted \
-u "${USER}":"${PASSWORD:-}" \
+ -H "Expect: 100-continue" \
-H "column_separator:|" \
-H "columns: ${table_columns[${table_name}]}" \
-T "${file}" \
@@ -190,6 +191,7 @@ for table_name in ${!table_columns[*]}; do
ret=$(curl \
--location-trusted \
-u "${USER}":"${PASSWORD:-}" \
+ -H "Expect: 100-continue" \
-H "label:${TXN_ID}_${FILE_ID}" \
-H "column_separator:|" \
-H "columns: ${table_columns[${table_name}]}" \
diff --git a/tools/tpch-tools/bin/load-tpch-data.sh
b/tools/tpch-tools/bin/load-tpch-data.sh
index fa21028f2e9..4eb9f9fdb3f 100755
--- a/tools/tpch-tools/bin/load-tpch-data.sh
+++ b/tools/tpch-tools/bin/load-tpch-data.sh
@@ -124,11 +124,11 @@ echo "DB: ${DB}"
function load_region() {
echo "$*"
if [[ -z ${TXN_ID} ]]; then
- curl -s --location-trusted -u "${USER}":"${PASSWORD}" -H
"column_separator:|" \
+ curl -s --location-trusted -u "${USER}":"${PASSWORD}" -H "Expect:
100-continue" -H "column_separator:|" \
-H "columns: r_regionkey, r_name, r_comment, temp" \
-T "$*"
http://"${FE_HOST}":"${FE_HTTP_PORT}"/api/"${DB}"/region/_stream_load
else
- curl -s --location-trusted -u "${USER}":"${PASSWORD}" -H
"column_separator:|" \
+ curl -s --location-trusted -u "${USER}":"${PASSWORD}" -H "Expect:
100-continue" -H "column_separator:|" \
-H "label:${TXN_ID}_region" \
-H "columns: r_regionkey, r_name, r_comment, temp" \
-T "$*"
http://"${FE_HOST}":"${FE_HTTP_PORT}"/api/"${DB}"/region/_stream_load
@@ -137,11 +137,11 @@ function load_region() {
function load_nation() {
echo "$*"
if [[ -z ${TXN_ID} ]]; then
- curl -s --location-trusted -u "${USER}":"${PASSWORD}" -H
"column_separator:|" \
+ curl -s --location-trusted -u "${USER}":"${PASSWORD}" -H "Expect:
100-continue" -H "column_separator:|" \
-H "columns: n_nationkey, n_name, n_regionkey, n_comment, temp" \
-T "$*"
http://"${FE_HOST}":"${FE_HTTP_PORT}"/api/"${DB}"/nation/_stream_load
else
- curl -s --location-trusted -u "${USER}":"${PASSWORD}" -H
"column_separator:|" \
+ curl -s --location-trusted -u "${USER}":"${PASSWORD}" -H "Expect:
100-continue" -H "column_separator:|" \
-H "label:${TXN_ID}_nation" \
-H "columns: n_nationkey, n_name, n_regionkey, n_comment, temp" \
-T "$*"
http://"${FE_HOST}":"${FE_HTTP_PORT}"/api/"${DB}"/nation/_stream_load
@@ -150,11 +150,11 @@ function load_nation() {
function load_supplier() {
echo "$*"
if [[ -z ${TXN_ID} ]]; then
- curl -s --location-trusted -u "${USER}":"${PASSWORD}" -H
"column_separator:|" \
+ curl -s --location-trusted -u "${USER}":"${PASSWORD}" -H "Expect:
100-continue" -H "column_separator:|" \
-H "columns: s_suppkey, s_name, s_address, s_nationkey, s_phone,
s_acctbal, s_comment, temp" \
-T "$*"
http://"${FE_HOST}":"${FE_HTTP_PORT}"/api/"${DB}"/supplier/_stream_load
else
- curl -s --location-trusted -u "${USER}":"${PASSWORD}" -H
"column_separator:|" \
+ curl -s --location-trusted -u "${USER}":"${PASSWORD}" -H "Expect:
100-continue" -H "column_separator:|" \
-H "label:${TXN_ID}_supplier" \
-H "columns: s_suppkey, s_name, s_address, s_nationkey, s_phone,
s_acctbal, s_comment, temp" \
-T "$*"
http://"${FE_HOST}":"${FE_HTTP_PORT}"/api/"${DB}"/supplier/_stream_load
@@ -163,11 +163,11 @@ function load_supplier() {
function load_customer() {
echo "$*"
if [[ -z ${TXN_ID} ]]; then
- curl -s --location-trusted -u "${USER}":"${PASSWORD}" -H
"column_separator:|" \
+ curl -s --location-trusted -u "${USER}":"${PASSWORD}" -H "Expect:
100-continue" -H "column_separator:|" \
-H "columns: c_custkey, c_name, c_address, c_nationkey, c_phone,
c_acctbal, c_mktsegment, c_comment, temp" \
-T "$*"
http://"${FE_HOST}":"${FE_HTTP_PORT}"/api/"${DB}"/customer/_stream_load
else
- curl -s --location-trusted -u "${USER}":"${PASSWORD}" -H
"column_separator:|" \
+ curl -s --location-trusted -u "${USER}":"${PASSWORD}" -H "Expect:
100-continue" -H "column_separator:|" \
-H "label:${TXN_ID}_customer" \
-H "columns: c_custkey, c_name, c_address, c_nationkey, c_phone,
c_acctbal, c_mktsegment, c_comment, temp" \
-T "$*"
http://"${FE_HOST}":"${FE_HTTP_PORT}"/api/"${DB}"/customer/_stream_load
@@ -176,11 +176,11 @@ function load_customer() {
function load_part() {
echo "$*"
if [[ -z ${TXN_ID} ]]; then
- curl -s --location-trusted -u "${USER}":"${PASSWORD}" -H
"column_separator:|" \
+ curl -s --location-trusted -u "${USER}":"${PASSWORD}" -H "Expect:
100-continue" -H "column_separator:|" \
-H "columns: p_partkey, p_name, p_mfgr, p_brand, p_type, p_size,
p_container, p_retailprice, p_comment, temp" \
-T "$*"
http://"${FE_HOST}":"${FE_HTTP_PORT}"/api/"${DB}"/part/_stream_load
else
- curl -s --location-trusted -u "${USER}":"${PASSWORD}" -H
"column_separator:|" \
+ curl -s --location-trusted -u "${USER}":"${PASSWORD}" -H "Expect:
100-continue" -H "column_separator:|" \
-H "label:${TXN_ID}_part" \
-H "columns: p_partkey, p_name, p_mfgr, p_brand, p_type, p_size,
p_container, p_retailprice, p_comment, temp" \
-T "$*"
http://"${FE_HOST}":"${FE_HTTP_PORT}"/api/"${DB}"/part/_stream_load
@@ -191,11 +191,11 @@ function load_partsupp() {
# shellcheck disable=SC2016,SC2124
local FILE_ID="${@//*./}"
if [[ -z ${TXN_ID} ]]; then
- curl -s --location-trusted -u "${USER}":"${PASSWORD}" -H
"column_separator:|" \
+ curl -s --location-trusted -u "${USER}":"${PASSWORD}" -H "Expect:
100-continue" -H "column_separator:|" \
-H "columns: ps_partkey, ps_suppkey, ps_availqty, ps_supplycost,
ps_comment, temp" \
-T "$*"
http://"${FE_HOST}":"${FE_HTTP_PORT}"/api/"${DB}"/partsupp/_stream_load
else
- curl -s --location-trusted -u "${USER}":"${PASSWORD}" -H
"column_separator:|" \
+ curl -s --location-trusted -u "${USER}":"${PASSWORD}" -H "Expect:
100-continue" -H "column_separator:|" \
-H "label:${TXN_ID}_partsupp_${FILE_ID}" \
-H "columns: ps_partkey, ps_suppkey, ps_availqty, ps_supplycost,
ps_comment, temp" \
-T "$*"
http://"${FE_HOST}":"${FE_HTTP_PORT}"/api/"${DB}"/partsupp/_stream_load
@@ -206,11 +206,11 @@ function load_orders() {
# shellcheck disable=SC2016,SC2124
local FILE_ID="${@//*./}"
if [[ -z ${TXN_ID} ]]; then
- curl -s --location-trusted -u "${USER}":"${PASSWORD}" -H
"column_separator:|" \
+ curl -s --location-trusted -u "${USER}":"${PASSWORD}" -H "Expect:
100-continue" -H "column_separator:|" \
-H "columns: o_orderkey, o_custkey, o_orderstatus, o_totalprice,
o_orderdate, o_orderpriority, o_clerk, o_shippriority, o_comment, temp" \
-T "$*"
http://"${FE_HOST}":"${FE_HTTP_PORT}"/api/"${DB}"/orders/_stream_load
else
- curl -s --location-trusted -u "${USER}":"${PASSWORD}" -H
"column_separator:|" \
+ curl -s --location-trusted -u "${USER}":"${PASSWORD}" -H "Expect:
100-continue" -H "column_separator:|" \
-H "label:${TXN_ID}_orders_${FILE_ID}" \
-H "columns: o_orderkey, o_custkey, o_orderstatus, o_totalprice,
o_orderdate, o_orderpriority, o_clerk, o_shippriority, o_comment, temp" \
-T "$*"
http://"${FE_HOST}":"${FE_HTTP_PORT}"/api/"${DB}"/orders/_stream_load
@@ -221,11 +221,11 @@ function load_lineitem() {
# shellcheck disable=SC2016,SC2124
local FILE_ID="${@//*./}"
if [[ -z ${TXN_ID} ]]; then
- curl -s --location-trusted -u "${USER}":"${PASSWORD}" -H
"column_separator:|" \
+ curl -s --location-trusted -u "${USER}":"${PASSWORD}" -H "Expect:
100-continue" -H "column_separator:|" \
-H "columns: l_orderkey, l_partkey, l_suppkey, l_linenumber,
l_quantity, l_extendedprice, l_discount, l_tax, l_returnflag,l_linestatus,
l_shipdate,l_commitdate,l_receiptdate,l_shipinstruct,l_shipmode,l_comment,temp"
\
-T "$*"
http://"${FE_HOST}":"${FE_HTTP_PORT}"/api/"${DB}"/lineitem/_stream_load
else
- curl -s --location-trusted -u "${USER}":"${PASSWORD}" -H
"column_separator:|" \
+ curl -s --location-trusted -u "${USER}":"${PASSWORD}" -H "Expect:
100-continue" -H "column_separator:|" \
-H "label:${TXN_ID}_lineitem_${FILE_ID}" \
-H "columns: l_orderkey, l_partkey, l_suppkey, l_linenumber,
l_quantity, l_extendedprice, l_discount, l_tax, l_returnflag,l_linestatus,
l_shipdate,l_commitdate,l_receiptdate,l_shipinstruct,l_shipmode,l_comment,temp"
\
-T "$*"
http://"${FE_HOST}":"${FE_HTTP_PORT}"/api/"${DB}"/lineitem/_stream_load
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
