This is an automated email from the ASF dual-hosted git repository.
xincheng pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git
The following commit(s) were added to refs/heads/dev by this push:
new 13dfa859b5 [Improvement-14371][Hivecli] Add support for hivevar
setting in hivecli task (#14423)
13dfa859b5 is described below
commit 13dfa859b59ae90b93a8ed29920c31b5aa05f30e
Author: Rick Cheng <[email protected]>
AuthorDate: Tue Jul 4 14:02:23 2023 +0800
[Improvement-14371][Hivecli] Add support for hivevar setting in hivecli
task (#14423)
---
.../plugin/task/hivecli/HiveCliTask.java | 71 +++++++++++++++++++---
.../plugin/task/hivecli/HiveCliTaskTest.java | 25 +++++++-
2 files changed, 85 insertions(+), 11 deletions(-)
diff --git
a/dolphinscheduler-task-plugin/dolphinscheduler-task-hivecli/src/main/java/org/apache/dolphinscheduler/plugin/task/hivecli/HiveCliTask.java
b/dolphinscheduler-task-plugin/dolphinscheduler-task-hivecli/src/main/java/org/apache/dolphinscheduler/plugin/task/hivecli/HiveCliTask.java
index 0aaf5f70b2..8ba8409567 100644
---
a/dolphinscheduler-task-plugin/dolphinscheduler-task-hivecli/src/main/java/org/apache/dolphinscheduler/plugin/task/hivecli/HiveCliTask.java
+++
b/dolphinscheduler-task-plugin/dolphinscheduler-task-hivecli/src/main/java/org/apache/dolphinscheduler/plugin/task/hivecli/HiveCliTask.java
@@ -18,8 +18,10 @@
package org.apache.dolphinscheduler.plugin.task.hivecli;
import static
org.apache.dolphinscheduler.plugin.task.api.TaskConstants.EXIT_CODE_FAILURE;
+import static
org.apache.dolphinscheduler.plugin.task.api.TaskConstants.RWXR_XR_X;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
+import org.apache.dolphinscheduler.common.utils.OSUtils;
import org.apache.dolphinscheduler.plugin.task.api.AbstractRemoteTask;
import org.apache.dolphinscheduler.plugin.task.api.ShellCommandExecutor;
import org.apache.dolphinscheduler.plugin.task.api.TaskCallBack;
@@ -31,12 +33,23 @@ import
org.apache.dolphinscheduler.plugin.task.api.model.TaskResponse;
import
org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters;
import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils;
+import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.StringUtils;
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardOpenOption;
+import java.nio.file.attribute.FileAttribute;
+import java.nio.file.attribute.PosixFilePermission;
+import java.nio.file.attribute.PosixFilePermissions;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Map;
+import java.util.Set;
public class HiveCliTask extends AbstractRemoteTask {
@@ -108,29 +121,41 @@ public class HiveCliTask extends AbstractRemoteTask {
final String type = hiveCliParameters.getHiveCliTaskExecutionType();
+ String sqlContent = "";
+ String resourceFileName = "";
// TODO: make sure type is not unknown
if (HiveCliConstants.TYPE_FILE.equals(type)) {
- args.add(HiveCliConstants.HIVE_CLI_EXECUTE_FILE);
final List<ResourceInfo> resourceInfos =
hiveCliParameters.getResourceList();
if (resourceInfos.size() > 1) {
log.warn("more than 1 files detected, use the first one by
default");
}
-
args.add(StringUtils.stripStart(resourceInfos.get(0).getResourceName(), "/"));
+ try {
+ resourceFileName = resourceInfos.get(0).getResourceName();
+ sqlContent = FileUtils.readFileToString(
+ new File(String.format("%s/%s",
taskExecutionContext.getExecutePath(), resourceFileName)),
+ StandardCharsets.UTF_8);
+ } catch (IOException e) {
+ log.error("read hive sql content from file {} error ",
resourceFileName, e);
+ throw new TaskException("read hive sql content error", e);
+ }
} else {
- final String script = hiveCliParameters.getHiveSqlScript();
- args.add(String.format(HiveCliConstants.HIVE_CLI_EXECUTE_SCRIPT,
script));
+ sqlContent = hiveCliParameters.getHiveSqlScript();
}
+ final Map<String, Property> paramsMap =
taskExecutionContext.getPrepareParamsMap();
+ sqlContent = ParameterUtils.convertParameterPlaceholders(sqlContent,
ParameterUtils.convert(paramsMap));
+ log.info("HiveCli sql content: {}", sqlContent);
+ String sqlFilePath = generateSqlScriptFile(sqlContent);
+
+ args.add(HiveCliConstants.HIVE_CLI_EXECUTE_FILE);
+ args.add(new File(sqlFilePath).getName());
final String hiveCliOptions = hiveCliParameters.getHiveCliOptions();
if (StringUtils.isNotEmpty(hiveCliOptions)) {
args.add(hiveCliOptions);
}
- final Map<String, Property> paramsMap =
taskExecutionContext.getPrepareParamsMap();
- final String command =
- ParameterUtils.convertParameterPlaceholders(String.join(" ",
args), ParameterUtils.convert(paramsMap));
-
+ String command = String.join(" ", args);
log.info("hiveCli task command: {}", command);
return command;
@@ -151,4 +176,34 @@ public class HiveCliTask extends AbstractRemoteTask {
}
}
+ protected String generateSqlScriptFile(String rawScript) {
+ String scriptFileName = String.format("%s/%s_node.sql",
taskExecutionContext.getExecutePath(),
+ taskExecutionContext.getTaskAppId());
+
+ File file = new File(scriptFileName);
+ Path path = file.toPath();
+
+ if (!Files.exists(path)) {
+ String script = rawScript.replaceAll("\\r\\n", "\n");
+
+ Set<PosixFilePermission> perms =
PosixFilePermissions.fromString(RWXR_XR_X);
+ FileAttribute<Set<PosixFilePermission>> attr =
PosixFilePermissions.asFileAttribute(perms);
+ try {
+ if (OSUtils.isWindows()) {
+ Files.createFile(path);
+ } else {
+ if (!file.getParentFile().exists()) {
+ file.getParentFile().mkdirs();
+ }
+ Files.createFile(path, attr);
+ }
+ Files.write(path, script.getBytes(),
StandardOpenOption.APPEND);
+ } catch (IOException e) {
+ log.error("generate hivecli sql script error", e);
+ throw new TaskException("generate hivecli sql script error",
e);
+ }
+ }
+ return scriptFileName;
+ }
+
}
diff --git
a/dolphinscheduler-task-plugin/dolphinscheduler-task-hivecli/src/test/java/org/apache/dolphinscheduler/plugin/task/hivecli/HiveCliTaskTest.java
b/dolphinscheduler-task-plugin/dolphinscheduler-task-hivecli/src/test/java/org/apache/dolphinscheduler/plugin/task/hivecli/HiveCliTaskTest.java
index 7064c23233..506651208b 100644
---
a/dolphinscheduler-task-plugin/dolphinscheduler-task-hivecli/src/test/java/org/apache/dolphinscheduler/plugin/task/hivecli/HiveCliTaskTest.java
+++
b/dolphinscheduler-task-plugin/dolphinscheduler-task-hivecli/src/test/java/org/apache/dolphinscheduler/plugin/task/hivecli/HiveCliTaskTest.java
@@ -17,6 +17,7 @@
package org.apache.dolphinscheduler.plugin.task.hivecli;
+import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.when;
@@ -24,12 +25,17 @@ import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
import org.apache.dolphinscheduler.plugin.task.api.model.ResourceInfo;
+import org.apache.commons.io.FileUtils;
+
import java.util.ArrayList;
import java.util.List;
+import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
+import org.mockito.MockedStatic;
import org.mockito.Mockito;
import org.mockito.junit.jupiter.MockitoExtension;
@@ -37,13 +43,25 @@ import org.mockito.junit.jupiter.MockitoExtension;
public class HiveCliTaskTest {
public static final String
EXPECTED_HIVE_CLI_TASK_EXECUTE_FROM_SCRIPT_COMMAND =
- "hive -e \"SHOW DATABASES;\"";
+ "hive -f 123_node.sql";
public static final String
EXPECTED_HIVE_CLI_TASK_EXECUTE_FROM_FILE_COMMAND =
- "hive -f sql_tasks/hive_task.sql";
+ "hive -f 123_node.sql";
public static final String EXPECTED_HIVE_CLI_TASK_EXECUTE_WITH_OPTIONS =
- "hive -e \"SHOW DATABASES;\" --verbose";
+ "hive -f 123_node.sql --verbose";
+
+ private MockedStatic<FileUtils> mockedStaticFileUtils;
+
+ @BeforeEach
+ public void setUp() {
+ mockedStaticFileUtils = Mockito.mockStatic(FileUtils.class);
+ }
+
+ @AfterEach
+ public void after() {
+ mockedStaticFileUtils.close();
+ }
@Test
public void hiveCliTaskExecuteSqlFromScript() throws Exception {
@@ -73,6 +91,7 @@ public class HiveCliTaskTest {
TaskExecutionContext taskExecutionContext =
Mockito.mock(TaskExecutionContext.class);
when(taskExecutionContext.getTaskParams()).thenReturn(hiveCliTaskParameters);
HiveCliTask hiveCliTask = spy(new HiveCliTask(taskExecutionContext));
+
doReturn("123_node.sql").when(hiveCliTask).generateSqlScriptFile(Mockito.any());
return hiveCliTask;
}
