mobiuschh commented on issue #12590:
URL:
https://github.com/apache/dolphinscheduler/issues/12590#issuecomment-1294664274
[LOG-PATH]:
/opt/dolphinscheduler/logs/20221028/7359692479072_2-17893-46126.log, [HOST]:
Host{address='dolphinscheduler-worker-2.dolphinscheduler-worker-headless:1234',
ip='dolphinscheduler-worker-2.dolphinscheduler-worker-headless', port=1234}
[INFO] 2022-10-28 11:37:37.070 +0800
[taskAppId=TASK-20221028-7359692479072_2-17893-46126] TaskLogLogger-class
org.apache.dolphinscheduler.plugin.task.dq.DataQualityTask:[83] - data quality
task params
{"localParams":[],"resourceList":[],"ruleId":1,"ruleInputParameter":{"check_type":"1","comparison_type":1,"comparison_name":"0","failure_strategy":"0","operator":"3","src_connector_type":0,"src_datasource_id":8,"src_field":"id","src_table":"ads_search_store_hotword_i_d","threshold":"0"},"sparkParameters":{"deployMode":"cluster","driverCores":1,"driverMemory":"512M","executorCores":2,"executorMemory":"2G","numExecutors":2,"others":"--conf
spark.yarn.maxAppAttempts=1"}}
[INFO] 2022-10-28 11:37:37.072 +0800
[taskAppId=TASK-20221028-7359692479072_2-17893-46126] TaskLogLogger-class
org.apache.dolphinscheduler.plugin.task.dq.DataQualityTask:[181] - data quality
task command: ${SPARK_HOME2}/bin/spark-submit --master yarn --deploy-mode
cluster --driver-cores 1 --driver-memory 512M --num-executors 2
--executor-cores 2 --executor-memory 2G --queue default --conf
spark.yarn.maxAppAttempts=1
/opt/dolphinscheduler/libs/dolphinscheduler-data-quality-dev-SNAPSHOT.jar
"{\"name\":\"$t(null_check)\",\"env\":{\"type\":\"batch\",\"config\":null},\"readers\":[{\"type\":\"JDBC\",\"config\":{\"database\":\"stat\",\"password\":\"***********\",\"driver\":\"com.mysql.cj.jdbc.Driver\",\"user\":\"bidata\",\"output_table\":\"stat_ads_search_store_hotword_i_d\",\"table\":\"ads_search_store_hotword_i_d\",\"url\":\"jdbc:mysql://am-wz99d8065a99x5f0w167320o.ads.aliyuncs.com:3306/stat?allowLoadLocalInfile=false&autoDeserialize=false&allowLocalInfile=false&allowUrlInLocalInfile=f
alse\"}
}],\"transformers\":[{\"type\":\"sql\",\"config\":{\"index\":1,\"output_table\":\"null_items\",\"sql\":\"SELECT
* FROM stat_ads_search_store_hotword_i_d WHERE (id is null or id = '') \"}
},{\"type\":\"sql\",\"config\":{\"index\":2,\"output_table\":\"null_count\",\"sql\":\"SELECT
COUNT(*) AS nulls FROM null_items\"}
}],\"writers\":[{\"type\":\"JDBC\",\"config\":{\"database\":\"dolphinscheduler\",\"password\":\"***************\",\"driver\":\"com.mysql.cj.jdbc.Driver\",\"user\":\"dolphinscheduler\",\"table\":\"t_ds_dq_execute_result\",\"url\":\"jdbc:mysql://rm-wz9s0d8l0d5139wh9.mysql.rds.aliyuncs.com:3306/dolphinscheduler?characterEncoding=utf8&allowLoadLocalInfile=false&autoDeserialize=false&allowLocalInfile=false&allowUrlInLocalInfile=false\",\"sql\":\"select
0 as rule_type,'$t(null_check)' as rule_name,0 as process_definition_id,17893
as process_instance_id,46126 as task_instance_id,null_count.nulls AS
statistics_value,0 AS comparison_value,1 AS comparison_type,1 as check_ty
pe,0 as threshold,3 as operator,0 as
failure_strategy,'hdfs://172.19.0.234:8020/user/dolphin/data_quality_error_data/0_17893_\u6570\u636E\u8D28\u91CF\u6D4B\u8BD5'
as error_output_path,'2022-10-28 11:37:37' as create_time,'2022-10-28
11:37:37' as update_time from null_count \"}
},{\"type\":\"JDBC\",\"config\":{\"database\":\"dolphinscheduler\",\"password\":\"***************\",\"driver\":\"com.mysql.cj.jdbc.Driver\",\"user\":\"dolphinscheduler\",\"table\":\"t_ds_dq_task_statistics_value\",\"url\":\"jdbc:mysql://rm-wz9s0d8l0d5139wh9.mysql.rds.aliyuncs.com:3306/dolphinscheduler?characterEncoding=utf8&allowLoadLocalInfile=false&autoDeserialize=false&allowLocalInfile=false&allowUrlInLocalInfile=false\",\"sql\":\"select
0 as process_definition_id,46126 as task_instance_id,1 as
rule_id,'3EHCYI2MVDDNF1CFUGCCEWXVTHXW+UB0PYME6EZPC3Y=' as
unique_code,'null_count.nulls'AS statistics_name,null_count.nulls AS
statistics_value,'2022-10-28 11:37:37' as data_time,'2022-10-28 11:37:37' as
create_time,
'2022-10-28 11:37:37' as update_time from null_count\"}
},{\"type\":\"hdfs_file\",\"config\":{\"path\":\"hdfs://172.19.0.234:8020/user/dolphin/data_quality_error_data/0_17893_\u6570\u636E\u8D28\u91CF\u6D4B\u8BD5\",\"input_table\":\"null_items\"}
}]}"
[INFO] 2022-10-28 11:37:37.073 +0800
[taskAppId=TASK-20221028-7359692479072_2-17893-46126] TaskLogLogger-class
org.apache.dolphinscheduler.plugin.task.dq.DataQualityTask:[85] - tenantCode
user:dolphin, task dir:17893_46126
[INFO] 2022-10-28 11:37:37.073 +0800
[taskAppId=TASK-20221028-7359692479072_2-17893-46126] TaskLogLogger-class
org.apache.dolphinscheduler.plugin.task.dq.DataQualityTask:[90] - create
command
file:/tmp/dolphinscheduler/exec/process/6355591552224/7359692479072_2/17893/46126/17893_46126.command
[INFO] 2022-10-28 11:37:37.073 +0800
[taskAppId=TASK-20221028-7359692479072_2-17893-46126] TaskLogLogger-class
org.apache.dolphinscheduler.plugin.task.dq.DataQualityTask:[116] - command :
#!/bin/sh
BASEDIR=$(cd `dirname $0`; pwd)
cd $BASEDIR
source /opt/dolphinscheduler/conf/dolphinscheduler_env.sh
${SPARK_HOME2}/bin/spark-submit --master yarn --deploy-mode cluster
--driver-cores 1 --driver-memory 512M --num-executors 2 --executor-cores 2
--executor-memory 2G --queue default --conf spark.yarn.maxAppAttempts=1
/opt/dolphinscheduler/libs/dolphinscheduler-data-quality-dev-SNAPSHOT.jar
"{\"name\":\"$t(null_check)\",\"env\":{\"type\":\"batch\",\"config\":null},\"readers\":[{\"type\":\"JDBC\",\"config\":{\"database\":\"stat\",\"password\":\"***********\",\"driver\":\"com.mysql.cj.jdbc.Driver\",\"user\":\"bidata\",\"output_table\":\"stat_ads_search_store_hotword_i_d\",\"table\":\"ads_search_store_hotword_i_d\",\"url\":\"jdbc:mysql://am-wz99d8065a99x5f0w167320o.ads.aliyuncs.com:3306/stat?allowLoadLocalInfile=false&autoDeserialize=false&allowLocalInfile=false&allowUrlInLocalInfile=false\"}
}],\"transformers\":[{\"type\":\"sql\",\"config\":{\"index\":1,\"output_table\":\"null_items\",\"sql\":\"SELECT
* FROM stat_ads_search_store_hotword_i_d WHERE (id is null or id = '') \"}
},{\"type\
":\"sql\",\"config\":{\"index\":2,\"output_table\":\"null_count\",\"sql\":\"SELECT
COUNT(*) AS nulls FROM null_items\"}
}],\"writers\":[{\"type\":\"JDBC\",\"config\":{\"database\":\"dolphinscheduler\",\"password\":\"***************\",\"driver\":\"com.mysql.cj.jdbc.Driver\",\"user\":\"dolphinscheduler\",\"table\":\"t_ds_dq_execute_result\",\"url\":\"jdbc:mysql://rm-wz9s0d8l0d5139wh9.mysql.rds.aliyuncs.com:3306/dolphinscheduler?characterEncoding=utf8&allowLoadLocalInfile=false&autoDeserialize=false&allowLocalInfile=false&allowUrlInLocalInfile=false\",\"sql\":\"select
0 as rule_type,'$t(null_check)' as rule_name,0 as process_definition_id,17893
as process_instance_id,46126 as task_instance_id,null_count.nulls AS
statistics_value,0 AS comparison_value,1 AS comparison_type,1 as check_type,0
as threshold,3 as operator,0 as
failure_strategy,'hdfs://172.19.0.234:8020/user/dolphin/data_quality_error_data/0_17893_\u6570\u636E\u8D28\u91CF\u6D4B\u8BD5'
as error_output_path,'2022-10-28 11:37:37'
as create_time,'2022-10-28 11:37:37' as update_time from null_count \"}
},{\"type\":\"JDBC\",\"config\":{\"database\":\"dolphinscheduler\",\"password\":\"***************\",\"driver\":\"com.mysql.cj.jdbc.Driver\",\"user\":\"dolphinscheduler\",\"table\":\"t_ds_dq_task_statistics_value\",\"url\":\"jdbc:mysql://rm-wz9s0d8l0d5139wh9.mysql.rds.aliyuncs.com:3306/dolphinscheduler?characterEncoding=utf8&allowLoadLocalInfile=false&autoDeserialize=false&allowLocalInfile=false&allowUrlInLocalInfile=false\",\"sql\":\"select
0 as process_definition_id,46126 as task_instance_id,1 as
rule_id,'3EHCYI2MVDDNF1CFUGCCEWXVTHXW+UB0PYME6EZPC3Y=' as
unique_code,'null_count.nulls'AS statistics_name,null_count.nulls AS
statistics_value,'2022-10-28 11:37:37' as data_time,'2022-10-28 11:37:37' as
create_time,'2022-10-28 11:37:37' as update_time from null_count\"}
},{\"type\":\"hdfs_file\",\"config\":{\"path\":\"hdfs://172.19.0.234:8020/user/dolphin/data_quality_error_data/0_17893_\u6570\u636E\u8D28\u91CF\u6D4B
\u8BD5\",\"input_table\":\"null_items\"} }]}"
[INFO] 2022-10-28 11:37:37.091 +0800
[taskAppId=TASK-20221028-7359692479072_2-17893-46126] TaskLogLogger-class
org.apache.dolphinscheduler.plugin.task.dq.DataQualityTask:[290] - task run
command: sudo -u dolphin sh
/tmp/dolphinscheduler/exec/process/6355591552224/7359692479072_2/17893/46126/17893_46126.command
[INFO] 2022-10-28 11:37:37.093 +0800
[taskAppId=TASK-20221028-7359692479072_2-17893-46126] TaskLogLogger-class
org.apache.dolphinscheduler.plugin.task.dq.DataQualityTask:[181] - process
start, process id is: 3214891
[INFO] 2022-10-28 11:37:37.106 +0800
[taskAppId=TASK-20221028-7359692479072_2-17893-46126] TaskLogLogger-class
org.apache.dolphinscheduler.plugin.task.dq.DataQualityTask:[205] - process has
exited, execute
path:/tmp/dolphinscheduler/exec/process/6355591552224/7359692479072_2/17893/46126,
processId:3214891 ,exitStatusCode:127 ,processWaitForStatus:true
,processExitValue:127
[INFO] 2022-10-28 11:37:38.094 +0800
[taskAppId=TASK-20221028-7359692479072_2-17893-46126] TaskLogLogger-class
org.apache.dolphinscheduler.plugin.task.dq.DataQualityTask:[63] - ->
/tmp/dolphinscheduler/exec/process/6355591552224/7359692479072_2/17893/46126/17893_46126.command:
line 5: /opt/soft/spark2/bin/spark-submit: No such file or directory
[INFO] 2022-10-28 11:37:38.095 +0800
[taskAppId=TASK-20221028-7359692479072_2-17893-46126] TaskLogLogger-class
org.apache.dolphinscheduler.plugin.task.dq.DataQualityTask:[57] -
FINALIZE_SESSION
@pandong2011
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
