This is an automated email from the ASF dual-hosted git repository.
leesf pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-hudi.git
The following commit(s) were added to refs/heads/master by this push:
new 8963a68 [HUDI-398]Add spark env set/get for spark launcher (#1096)
8963a68 is described below
commit 8963a68e6a3f4875de6787dfc206543e9ca824d9
Author: hongdd <jn_...@163.com>
AuthorDate: Sat Dec 14 14:13:00 2019 +0800
[HUDI-398]Add spark env set/get for spark launcher (#1096)
---
.../apache/hudi/cli/commands/SparkEnvCommand.java | 68 ++++++++++++++++++++++
.../java/org/apache/hudi/cli/utils/SparkUtil.java | 6 +-
2 files changed, 72 insertions(+), 2 deletions(-)
diff --git
a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/SparkEnvCommand.java
b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/SparkEnvCommand.java
new file mode 100644
index 0000000..b7e6eb0
--- /dev/null
+++ b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/SparkEnvCommand.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.cli.commands;
+
+import org.apache.hudi.cli.HoodiePrintHelper;
+
+import org.springframework.shell.core.CommandMarker;
+import org.springframework.shell.core.annotation.CliCommand;
+import org.springframework.shell.core.annotation.CliOption;
+import org.springframework.stereotype.Component;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * CLI command to set and show spark launcher init env.
+ */
+@Component
+public class SparkEnvCommand implements CommandMarker {
+
+ public static Map<String, String> env = new HashMap<String, String>();
+
+ @CliCommand(value = "set", help = "Set spark launcher env to cli")
+ public void setEnv(@CliOption(key = {"conf"}, help = "Env config to be set")
final String confMap)
+ throws IllegalArgumentException {
+ String[] map = confMap.split("=");
+ if (map.length != 2) {
+ throw new IllegalArgumentException("Illegal set parameter, please use
like [set --conf SPARK_HOME=/usr/etc/spark]");
+ }
+ env.put(map[0].trim(), map[1].trim());
+ }
+
+ @CliCommand(value = "show env all", help = "Show spark launcher envs")
+ public String showAllEnv() {
+ String[][] rows = new String[env.size()][2];
+ int i = 0;
+ for (String key: env.keySet()) {
+ rows[i] = new String[]{key, env.get(key)};
+ i++;
+ }
+ return HoodiePrintHelper.print(new String[] {"key", "value"}, rows);
+ }
+
+ @CliCommand(value = "show env", help = "Show spark launcher env by key")
+ public String showEnvByKey(@CliOption(key = {"key"}, help = "Which env conf
want to show") final String key) {
+ if (key == null || key.isEmpty()) {
+ return showAllEnv();
+ } else {
+ return HoodiePrintHelper.print(new String[] {"key", "value"}, new
String[][]{new String[]{key, env.get(key)}});
+ }
+ }
+}
diff --git a/hudi-cli/src/main/java/org/apache/hudi/cli/utils/SparkUtil.java
b/hudi-cli/src/main/java/org/apache/hudi/cli/utils/SparkUtil.java
index 3a21e56..5b5a3f5 100644
--- a/hudi-cli/src/main/java/org/apache/hudi/cli/utils/SparkUtil.java
+++ b/hudi-cli/src/main/java/org/apache/hudi/cli/utils/SparkUtil.java
@@ -19,6 +19,7 @@
package org.apache.hudi.cli.utils;
import org.apache.hudi.HoodieWriteClient;
+import org.apache.hudi.cli.commands.SparkEnvCommand;
import org.apache.hudi.cli.commands.SparkMain;
import org.apache.hudi.common.util.FSUtils;
import org.apache.hudi.common.util.StringUtils;
@@ -30,6 +31,7 @@ import org.apache.spark.launcher.SparkLauncher;
import java.io.File;
import java.net.URISyntaxException;
+import java.util.Map;
/**
* Utility functions dealing with Spark.
@@ -45,13 +47,13 @@ public class SparkUtil {
public static SparkLauncher initLauncher(String propertiesFile) throws
URISyntaxException {
String currentJar = new
File(SparkUtil.class.getProtectionDomain().getCodeSource().getLocation().toURI().getPath())
.getAbsolutePath();
+ Map<String, String> env = SparkEnvCommand.env;
SparkLauncher sparkLauncher =
- new
SparkLauncher().setAppResource(currentJar).setMainClass(SparkMain.class.getName());
+ new
SparkLauncher(env).setAppResource(currentJar).setMainClass(SparkMain.class.getName());
if (!StringUtils.isNullOrEmpty(propertiesFile)) {
sparkLauncher.setPropertiesFile(propertiesFile);
}
-
File libDirectory = new File(new File(currentJar).getParent(), "lib");
for (String library : libDirectory.list()) {
sparkLauncher.addJar(new File(libDirectory, library).getAbsolutePath());
