calvinjiang opened a new pull request #6531: URL: https://github.com/apache/dolphinscheduler/pull/6531
<!--Thanks very much for contributing to Apache DolphinScheduler. Please review https://dolphinscheduler.apache.org/en-us/community/development/pull-request.html before opening a pull request.--> ## Purpose of the pull request This PR will close the issue #6529. <!--(For example: This pull request adds checkstyle plugin).--> ## Brief change log 1.The new python task uses the 'ShellCommandExecutor' to execute task. 2.First create a python script file that contains the task script. 3.And create a shell file will execute the python script file. 4.So you can get environment variables in the 'dolphinscheduler_env.sh'. <!--*(for example:)* - *Add maven-checkstyle-plugin to root pom.xml* --> ## Verify this pull request - *Manually verified the change by testing locally.* Create an environment:  Create a python task called PythonTask:  Save this task and run it, you will get the result as follows: ``` 2021-10-14 17:25:20.776 INFO 93069 --- [pool-7-thread-4] o.a.d.s.w.p.TaskExecuteProcessor : received command : TaskExecuteRequestCommand{taskExecutionContext='{"taskInstanceId":59,"taskName":"PythonTask","firstSubmitTime":"2021-10-14 17:25:20","startTime":null,"taskType":"PYTHON","host":null,"executePath":null,"logPath":null,"taskJson":null,"processId":0,"processDefineCode":806053553881088,"processDefineVersion":2,"appIds":null,"processInstanceId":45,"scheduleTime":null,"globalParams":null,"executorId":1,"cmdTypeIfComplement":0,"tenantCode":"calvin","queue":"default","projectCode":723800319311872,"taskParams":"{\"resourceList\":[],\"localParams\":[{\"prop\":\"current_datetime\",\"direct\":\"IN\",\"type\":\"VARCHAR\",\"value\":\"${system.datetime}\"}],\"rawScript\":\"import os\\nimport sys\\n\\nprint(\\\"Python version:%s\\\\n\\\" %(sys.version))\\n\\nprint(\\\"HADOOP_HOME=%s\\\\n\\\" %(os.getenv('HADOOP_HOME')))\\nprint(\\\"HADOOP_CONF_DIR=%s\\\\n\\\" %(os.getenv('HADOOP_ CONF_DIR')))\\nprint(\\\"SPARK_HOME1=%s\\\\n\\\" %(os.getenv('SPARK_HOME1')))\\nprint(\\\"SPARK_HOME2=%s\\\\n\\\" %(os.getenv('SPARK_HOME2')))\\nprint(\\\"PYTHON_HOME=%s\\\\n\\\" %(os.getenv('PYTHON_HOME')))\\nprint(\\\"JAVA_HOME=%s\\\\n\\\" %(os.getenv('JAVA_HOME')))\\nprint(\\\"HIVE_HOME=%s\\\\n\\\" %(os.getenv('HIVE_HOME')))\\nprint(\\\"FLINK_HOME=%s\\\\n\\\" %(os.getenv('FLINK_HOME')))\\nprint(\\\"JAVA_HOME=%s\\\\n\\\" %(os.getenv('JAVA_HOME')))\\nprint(\\\"DATAX_HOME=%s\\\\n\\\" %(os.getenv('DATAX_HOME')))\\nprint(\\\"PATH=%s\\\\n\\\" %(os.getenv('PATH')))\\n\\nprint(\\\"current_datetime=%s\\\\n\\\" %('${current_datetime}'))\\n\",\"dependence\":{},\"conditionResult\":{\"successNode\":[\"\"],\"failedNode\":[\"\"]},\"waitStartTimeout\":{}}","envFile":null,"environmentConfig":"export HADOOP_HOME=/opt/soft/hadoop\nexport HADOOP_CONF_DIR=/opt/soft/hadoop/etc/hadoop\nexport SPARK_HOME1=/opt/soft/spark1\nexport SPARK_HOME2=/opt/soft/spark2\nexport PYTHON_HOME=/usr/\n#export PYTHON_HOM E=/opt/soft/python\nexport JAVA_HOME=/opt/soft/java\nexport HIVE_HOME=/opt/soft/hive\nexport FLINK_HOME=/opt/soft/flink\nexport DATAX_HOME=/opt/soft/datax\nexport PATH=$HADOOP_HOME/bin:$SPARK_HOME1/bin:$SPARK_HOME2/bin:$PYTHON_HOME/bin:$JAVA_HOME/bin:$HIVE_HOME/bin:$FLINK_HOME/bin:$DATAX_HOME/bin:$PATH","definedParams":null,"taskAppId":null,"taskTimeoutStrategy":null,"taskTimeout":2147483647,"workerGroup":"default","delayTime":0,"currentExecutionStatus":null,"resources":{},"sqlTaskExecutionContext":{"warningGroupId":0,"connectionParams":null,"udfFuncTenantCodeMap":null},"dataxTaskExecutionContext":{"dataSourceId":0,"sourcetype":0,"sourceConnectionParams":null,"dataTargetId":0,"targetType":0,"targetConnectionParams":null},"dependenceTaskExecutionContext":null,"sqoopTaskExecutionContext":{"dataSourceId":0,"sourcetype":0,"sourceConnectionParams":null,"dataTargetId":0,"targetType":0,"targetConnectionParams":null},"varPool":null,"dryRun":0,"paramsMap":null,"procedureTaskExecutionContext" :{"connectionParams":null}}'} 2021-10-14 17:25:20.779 INFO 93069 --- [pool-7-thread-4] o.a.d.s.w.p.TaskExecuteProcessor : task instance local execute path : /tmp/dolphinscheduler/exec/process/723800319311872/806053553881088_2/45/59 2021-10-14 17:25:20.780 INFO 93069 --- [pool-7-thread-4] o.a.d.common.utils.FileUtils : create dir success /tmp/dolphinscheduler/exec/process/723800319311872/806053553881088_2/45/59 2021-10-14 17:25:20.785 INFO 93069 --- [-Execute-Thread] o.a.d.s.worker.runner.TaskExecuteThread : script path : /tmp/dolphinscheduler/exec/process/723800319311872/806053553881088_2/45/59 2021-10-14 17:25:20.875 INFO 93069 --- [-Execute-Thread] o.a.d.s.worker.runner.TaskExecuteThread : the task begins to execute. task instance id: 59 2021-10-14 17:25:20.877 INFO 93069 --- [-Execute-Thread] [taskAppId=TASK-806053553881088_2-45-59] : python task params {"resourceList":[],"localParams":[{"prop":"current_datetime","direct":"IN","type":"VARCHAR","value":"${system.datetime}"}],"rawScript":"import os\nimport sys\n\nprint(\"Python version:%s\\n\" %(sys.version))\n\nprint(\"HADOOP_HOME=%s\\n\" %(os.getenv('HADOOP_HOME')))\nprint(\"HADOOP_CONF_DIR=%s\\n\" %(os.getenv('HADOOP_CONF_DIR')))\nprint(\"SPARK_HOME1=%s\\n\" %(os.getenv('SPARK_HOME1')))\nprint(\"SPARK_HOME2=%s\\n\" %(os.getenv('SPARK_HOME2')))\nprint(\"PYTHON_HOME=%s\\n\" %(os.getenv('PYTHON_HOME')))\nprint(\"JAVA_HOME=%s\\n\" %(os.getenv('JAVA_HOME')))\nprint(\"HIVE_HOME=%s\\n\" %(os.getenv('HIVE_HOME')))\nprint(\"FLINK_HOME=%s\\n\" %(os.getenv('FLINK_HOME')))\nprint(\"JAVA_HOME=%s\\n\" %(os.getenv('JAVA_HOME')))\nprint(\"DATAX_HOME=%s\\n\" %(os.getenv('DATAX_HOME')))\nprint(\"PATH=%s\\n\" %(os.getenv('PATH')))\n\nprint(\"current_datetime=%s\\n\" %('${current_ datetime}'))\n","dependence":{},"conditionResult":{"successNode":[""],"failedNode":[""]},"waitStartTimeout":{}} 2021-10-14 17:25:20.888 INFO 93069 --- [-Execute-Thread] [taskAppId=TASK-806053553881088_2-45-59] : raw python script : import os import sys print("Python version:%s\n" %(sys.version)) print("HADOOP_HOME=%s\n" %(os.getenv('HADOOP_HOME'))) print("HADOOP_CONF_DIR=%s\n" %(os.getenv('HADOOP_CONF_DIR'))) print("SPARK_HOME1=%s\n" %(os.getenv('SPARK_HOME1'))) print("SPARK_HOME2=%s\n" %(os.getenv('SPARK_HOME2'))) print("PYTHON_HOME=%s\n" %(os.getenv('PYTHON_HOME'))) print("JAVA_HOME=%s\n" %(os.getenv('JAVA_HOME'))) print("HIVE_HOME=%s\n" %(os.getenv('HIVE_HOME'))) print("FLINK_HOME=%s\n" %(os.getenv('FLINK_HOME'))) print("JAVA_HOME=%s\n" %(os.getenv('JAVA_HOME'))) print("DATAX_HOME=%s\n" %(os.getenv('DATAX_HOME'))) print("PATH=%s\n" %(os.getenv('PATH'))) print("current_datetime=%s\n" %('${current_datetime}')) 2021-10-14 17:25:20.889 INFO 93069 --- [-Execute-Thread] [taskAppId=TASK-806053553881088_2-45-59] : tenantCode :calvin, task dir:/tmp/dolphinscheduler/exec/process/723800319311872/806053553881088_2/45/59 2021-10-14 17:25:20.889 INFO 93069 --- [-Execute-Thread] [taskAppId=TASK-806053553881088_2-45-59] : generate python script file:/tmp/dolphinscheduler/exec/process/723800319311872/806053553881088_2/45/59/py_45_59.py 2021-10-14 17:25:20.889 INFO 93069 --- [-Execute-Thread] [taskAppId=TASK-806053553881088_2-45-59] : #-*- encoding=utf8 -*- import os import sys print("Python version:%s\n" %(sys.version)) print("HADOOP_HOME=%s\n" %(os.getenv('HADOOP_HOME'))) print("HADOOP_CONF_DIR=%s\n" %(os.getenv('HADOOP_CONF_DIR'))) print("SPARK_HOME1=%s\n" %(os.getenv('SPARK_HOME1'))) print("SPARK_HOME2=%s\n" %(os.getenv('SPARK_HOME2'))) print("PYTHON_HOME=%s\n" %(os.getenv('PYTHON_HOME'))) print("JAVA_HOME=%s\n" %(os.getenv('JAVA_HOME'))) print("HIVE_HOME=%s\n" %(os.getenv('HIVE_HOME'))) print("FLINK_HOME=%s\n" %(os.getenv('FLINK_HOME'))) print("JAVA_HOME=%s\n" %(os.getenv('JAVA_HOME'))) print("DATAX_HOME=%s\n" %(os.getenv('DATAX_HOME'))) print("PATH=%s\n" %(os.getenv('PATH'))) print("current_datetime=%s\n" %('20211014172520')) 2021-10-14 17:25:20.890 INFO 93069 --- [-Execute-Thread] [taskAppId=TASK-806053553881088_2-45-59] : tenantCode user:calvin, task dir:45_59 2021-10-14 17:25:20.890 INFO 93069 --- [-Execute-Thread] [taskAppId=TASK-806053553881088_2-45-59] : create command file:/tmp/dolphinscheduler/exec/process/723800319311872/806053553881088_2/45/59/45_59.command 2021-10-14 17:25:20.890 INFO 93069 --- [-Execute-Thread] [taskAppId=TASK-806053553881088_2-45-59] : command : #!/bin/sh BASEDIR=$(cd `dirname $0`; pwd) cd $BASEDIR export HADOOP_HOME=/opt/soft/hadoop export HADOOP_CONF_DIR=/opt/soft/hadoop/etc/hadoop export SPARK_HOME1=/opt/soft/spark1 export SPARK_HOME2=/opt/soft/spark2 export PYTHON_HOME=/usr/ #export PYTHON_HOME=/opt/soft/python export JAVA_HOME=/opt/soft/java export HIVE_HOME=/opt/soft/hive export FLINK_HOME=/opt/soft/flink export DATAX_HOME=/opt/soft/datax export PATH=$HADOOP_HOME/bin:$SPARK_HOME1/bin:$SPARK_HOME2/bin:$PYTHON_HOME/bin:$JAVA_HOME/bin:$HIVE_HOME/bin:$FLINK_HOME/bin:$DATAX_HOME/bin:$PATH python /tmp/dolphinscheduler/exec/process/723800319311872/806053553881088_2/45/59/py_45_59.py 2021-10-14 17:25:20.903 INFO 93069 --- [-Execute-Thread] [taskAppId=TASK-806053553881088_2-45-59] : task run command: sudo -u calvin sh /tmp/dolphinscheduler/exec/process/723800319311872/806053553881088_2/45/59/45_59.command 2021-10-14 17:25:20.904 INFO 93069 --- [-Execute-Thread] [taskAppId=TASK-806053553881088_2-45-59] : process start, process id is: 93114 2021-10-14 17:25:21.033 INFO 93069 --- [-Execute-Thread] [taskAppId=TASK-806053553881088_2-45-59] : process has exited, execute path:/tmp/dolphinscheduler/exec/process/723800319311872/806053553881088_2/45/59, processId:93114 ,exitStatusCode:0 ,processWaitForStatus:true ,processExitValue:0 2021-10-14 17:25:21.034 INFO 93069 --- [-Execute-Thread] o.a.d.s.worker.runner.TaskExecuteThread : task instance id : 59,task final status : SUCCESS 2021-10-14 17:25:21.035 INFO 93069 --- [-Execute-Thread] o.a.d.s.worker.runner.TaskExecuteThread : develop mode is: false 2021-10-14 17:25:21.036 INFO 93069 --- [-Execute-Thread] o.a.d.s.worker.runner.TaskExecuteThread : exec local path: /tmp/dolphinscheduler/exec/process/723800319311872/806053553881088_2/45/59 cleared. 2021-10-14 17:25:21.905 INFO 93069 --- [skLogInfo-45_59] [taskAppId=TASK-806053553881088_2-45-59] : -> welcome to use bigdata scheduling system... Python version:2.7.16 (default, Jun 18 2021, 03:23:53) [GCC Apple LLVM 12.0.5 (clang-1205.0.19.59.6) [+internal-os, ptrauth-isa=deploy HADOOP_HOME=/opt/soft/hadoop HADOOP_CONF_DIR=/opt/soft/hadoop/etc/hadoop SPARK_HOME1=/opt/soft/spark1 SPARK_HOME2=/opt/soft/spark2 PYTHON_HOME=/usr/ JAVA_HOME=/opt/soft/java HIVE_HOME=/opt/soft/hive FLINK_HOME=/opt/soft/flink JAVA_HOME=/opt/soft/java DATAX_HOME=/opt/soft/datax PATH=/opt/soft/hadoop/bin:/opt/soft/spark1/bin:/opt/soft/spark2/bin:/usr//bin:/opt/soft/java/bin:/opt/soft/hive/bin:/opt/soft/flink/bin:/opt/soft/datax/bin:/usr/local/Cellar/pyenv/1.2.16/libexec:/usr/local/opt/nvm/versions/node/v12.4.0/bin:/usr/local/Cellar/pyenv-virtualenv/1.1.3/shims:/Users/calvin/.pyenv/shims:/usr/local/bin:/usr/local/cuda/bin:/Users/calvin/opt/hadoop-2.6.5/bin:/Users/calvin/redis-4.0.8/src:/Users/calvin/btrace/bin:/Library/java/JavaVirtualMachines/jdk1.8.0_191.jdk/Contents/Home/bin:/usr/local/opt/maven3/bin:/Users/calvin/phantomjs-2.1.1/bin:/Users/calvin/sysdir/apache-flume-1.7.0/bin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin:/Library/Apple/usr/bin:/Library/Frameworks/Mono.framework/Versions/Current/Commands:/Applications/Wireshark.app/Contents/MacOS:/opt/apache-tinkerpop-gremlin-console-3.4.3/bin:/usr/local/Cellar/postgresql@10/10.11/bin:/usr/local/sbin:/opt/gradle/gradle-7.2/bin:/opt/javacc-javacc-7.0.10/scripts current_datetime=20211014172520 2021-10-14 17:25:21.905 INFO 93069 --- [skLogInfo-45_59] [taskAppId=TASK-806053553881088_2-45-59] : FINALIZE_SESSION ``` -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
