This is an automated email from the ASF dual-hosted git repository.
lmccay pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/knox.git
The following commit(s) were added to refs/heads/master by this push:
new b93cc1c KNOX-2240 - KnoxShell Custom Command for WEBHDFS Use (#296)
b93cc1c is described below
commit b93cc1c68afa0e773027bfa2745f1d37f376bca1
Author: lmccay <[email protected]>
AuthorDate: Wed Mar 25 01:08:37 2020 -0400
KNOX-2240 - KnoxShell Custom Command for WEBHDFS Use (#296)
* KNOX-2240 - KnoxShell Custom Command for WEBHDFS Use
Change-Id: I817969b5131dd08e5a7a174c73aacb13e9027e4a
* Prompt to overwrite on put
Change-Id: I110037204fcd1cd556d0336321e309cd8e8dff08
* improve csv builder
Change-Id: I5b86694ea374ed2f78dfc049402f40e7de1fa508
* default to-path in an :fs put command to home directory and same filename
as source
Change-Id: I8e75b8aadd81625a1c4aff05851f1557496f4ef9
* fix class clast issue
Change-Id: Ifb14d1ca93ca3702a06ff744b7e79f524ea0b5d8
* KNOX-2240 - address review comments
Change-Id: Ibb4e4c3c3590aafb22d618b602de96e2cb4eb9f0
* KNOX-2240 - fix PMD failure
Change-Id: I1d47c0aa60f6c409fb59b16c8b9e3f57070be8ca
---
.../org/apache/knox/gateway/shell/KnoxSession.java | 38 ++
.../java/org/apache/knox/gateway/shell/Shell.java | 2 +
.../shell/commands/AbstractKnoxShellCommand.java | 9 +
.../shell/commands/AbstractSQLCommandSupport.java | 8 -
.../knox/gateway/shell/commands/CSVCommand.java | 18 +-
.../gateway/shell/commands/WebHDFSCommand.java | 426 +++++++++++++++++++++
.../org/apache/knox/gateway/shell/hdfs/Ls.java | 2 +-
.../org/apache/knox/gateway/shell/hdfs/Mkdir.java | 2 +-
.../org/apache/knox/gateway/shell/hdfs/Put.java | 2 +-
.../org/apache/knox/gateway/shell/hdfs/Rm.java | 2 +-
.../shell/table/CSVKnoxShellTableBuilder.java | 50 ++-
.../knox/gateway/shell/table/KnoxShellTable.java | 10 +-
.../shell/table/KnoxShellTableFileUtils.java | 47 +++
.../shell/table/KnoxShellTableJSONSerializer.java | 23 +-
.../shell/table/KnoxShellTableRenderer.java | 16 +-
.../gateway/shell/table/KnoxShellTableTest.java | 8 +
.../org/apache/knox/gateway/util/JsonUtils.java | 15 +
.../apache/knox/gateway/util/JsonUtilsTest.java | 45 ++-
18 files changed, 666 insertions(+), 57 deletions(-)
diff --git
a/gateway-shell/src/main/java/org/apache/knox/gateway/shell/KnoxSession.java
b/gateway-shell/src/main/java/org/apache/knox/gateway/shell/KnoxSession.java
index 563dcd8..a3d691d 100644
--- a/gateway-shell/src/main/java/org/apache/knox/gateway/shell/KnoxSession.java
+++ b/gateway-shell/src/main/java/org/apache/knox/gateway/shell/KnoxSession.java
@@ -141,6 +141,7 @@ public class KnoxSession implements Closeable {
private static final String KNOXSQLHISTORIES_JSON = "knoxsqlhistories.json";
private static final String KNOXDATASOURCES_JSON = "knoxdatasources.json";
+ private static final String KNOXMOUNTPOINTS_JSON = "knoxmountpoints.json";
public Map<String, String> getHeaders() {
return headers;
@@ -607,6 +608,16 @@ public class KnoxSession implements Closeable {
* @param map to persist
*/
public static <T> void persistDataSourcesToKnoxShell(String fileName,
Map<String, T> map) {
+ persistMapToKnoxShell(fileName, map);
+ }
+
+ /**
+ * Persist provided Map to a file within the {user.home}/.knoxshell directory
+ * @param <T> type of the value in the map
+ * @param fileName of persisted file
+ * @param map to persist
+ */
+ public static <T> void persistMapToKnoxShell(String fileName, Map<String, T>
map) {
String s = JsonUtils.renderAsJsonString(map);
String home = System.getProperty("user.home");
try {
@@ -641,6 +652,10 @@ public class KnoxSession implements Closeable {
persistDataSourcesToKnoxShell(KNOXDATASOURCES_JSON, datasources);
}
+ public static void persistMountPoints(Map<String, String> mounts) {
+ persistMapToKnoxShell(KNOXMOUNTPOINTS_JSON, mounts);
+ }
+
/**
* Load and return a map of datasource names to sql commands
* from the {user.home}/.knoxshell/knoxsqlhistories.json file.
@@ -661,6 +676,20 @@ public class KnoxSession implements Closeable {
return sqlHistories;
}
+ public static Map<String, String> loadMountPoints() throws IOException {
+ Map<String, String> mounts = new HashMap<>();
+ String home = System.getProperty("user.home");
+
+ File mountFile = new File(
+ home + File.separator +
+ ".knoxshell" + File.separator + KNOXMOUNTPOINTS_JSON);
+ if (mountFile.exists()) {
+ String json = readFileToString(mountFile);
+ mounts = getMapFromJsonString(json);
+ }
+ return mounts;
+ }
+
private static String readFileToString(File file) throws IOException {
String content = null;
@@ -705,6 +734,15 @@ public class KnoxSession implements Closeable {
return obj;
}
+ public static <T> Map<String, T> getMapFromJsonString(String json) throws
IOException {
+ Map<String, T> obj;
+ JsonFactory factory = new JsonFactory();
+ ObjectMapper mapper = new ObjectMapper(factory);
+ TypeReference<Map<String, T>> typeRef = new TypeReference<Map<String,
T>>() {};
+ obj = mapper.readValue(json, typeRef);
+ return obj;
+ }
+
public static Map<String, KnoxDataSource>
getMapOfDataSourcesFromJsonString(String json) throws IOException {
Map<String, KnoxDataSource> obj;
JsonFactory factory = new JsonFactory();
diff --git
a/gateway-shell/src/main/java/org/apache/knox/gateway/shell/Shell.java
b/gateway-shell/src/main/java/org/apache/knox/gateway/shell/Shell.java
index 6542dcb..c4a0be6 100644
--- a/gateway-shell/src/main/java/org/apache/knox/gateway/shell/Shell.java
+++ b/gateway-shell/src/main/java/org/apache/knox/gateway/shell/Shell.java
@@ -23,6 +23,7 @@ import
org.apache.knox.gateway.shell.commands.AbstractSQLCommandSupport;
import org.apache.knox.gateway.shell.commands.CSVCommand;
import org.apache.knox.gateway.shell.commands.DataSourceCommand;
import org.apache.knox.gateway.shell.commands.SelectCommand;
+import org.apache.knox.gateway.shell.commands.WebHDFSCommand;
import org.apache.knox.gateway.shell.hbase.HBase;
import org.apache.knox.gateway.shell.hdfs.Hdfs;
import org.apache.knox.gateway.shell.job.Job;
@@ -96,6 +97,7 @@ public class Shell {
shell.register(new SelectCommand(shell));
shell.register(new DataSourceCommand(shell));
shell.register(new CSVCommand(shell));
+ shell.register(new WebHDFSCommand(shell));
shell.run( null );
}
}
diff --git
a/gateway-shell/src/main/java/org/apache/knox/gateway/shell/commands/AbstractKnoxShellCommand.java
b/gateway-shell/src/main/java/org/apache/knox/gateway/shell/commands/AbstractKnoxShellCommand.java
index 3fa400e..a24edad 100644
---
a/gateway-shell/src/main/java/org/apache/knox/gateway/shell/commands/AbstractKnoxShellCommand.java
+++
b/gateway-shell/src/main/java/org/apache/knox/gateway/shell/commands/AbstractKnoxShellCommand.java
@@ -18,6 +18,9 @@
package org.apache.knox.gateway.shell.commands;
import java.util.List;
+
+import org.apache.knox.gateway.shell.CredentialCollectionException;
+import org.apache.knox.gateway.shell.CredentialCollector;
import org.codehaus.groovy.tools.shell.CommandSupport;
import org.codehaus.groovy.tools.shell.Groovysh;
@@ -43,4 +46,10 @@ public abstract class AbstractKnoxShellCommand extends
CommandSupport {
}
return variableName;
}
+
+ protected CredentialCollector login() throws CredentialCollectionException {
+ KnoxLoginDialog dlg = new KnoxLoginDialog();
+ dlg.collect();
+ return dlg;
+ }
}
\ No newline at end of file
diff --git
a/gateway-shell/src/main/java/org/apache/knox/gateway/shell/commands/AbstractSQLCommandSupport.java
b/gateway-shell/src/main/java/org/apache/knox/gateway/shell/commands/AbstractSQLCommandSupport.java
index eaeea01..d39699f 100644
---
a/gateway-shell/src/main/java/org/apache/knox/gateway/shell/commands/AbstractSQLCommandSupport.java
+++
b/gateway-shell/src/main/java/org/apache/knox/gateway/shell/commands/AbstractSQLCommandSupport.java
@@ -25,8 +25,6 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import org.apache.knox.gateway.shell.CredentialCollectionException;
-import org.apache.knox.gateway.shell.CredentialCollector;
import org.apache.knox.gateway.shell.KnoxDataSource;
import org.apache.knox.gateway.shell.KnoxSession;
import org.apache.knox.gateway.shell.jdbc.JDBCUtils;
@@ -202,10 +200,4 @@ public abstract class AbstractSQLCommandSupport extends
AbstractKnoxShellCommand
}
});
}
-
- protected CredentialCollector login() throws CredentialCollectionException {
- KnoxLoginDialog dlg = new KnoxLoginDialog();
- dlg.collect();
- return dlg;
- }
}
\ No newline at end of file
diff --git
a/gateway-shell/src/main/java/org/apache/knox/gateway/shell/commands/CSVCommand.java
b/gateway-shell/src/main/java/org/apache/knox/gateway/shell/commands/CSVCommand.java
index 22aad2c..b8c3619 100644
---
a/gateway-shell/src/main/java/org/apache/knox/gateway/shell/commands/CSVCommand.java
+++
b/gateway-shell/src/main/java/org/apache/knox/gateway/shell/commands/CSVCommand.java
@@ -49,10 +49,24 @@ public class CSVCommand extends AbstractKnoxShellCommand {
try {
if (withHeaders) {
- table = KnoxShellTable.builder().csv().withHeaders().url(url);
+ if (url.startsWith("$")) {
+ // a knoxshell variable is a csv file as a string
+ String csvString = (String) getVariables().get(url.substring(1));
+ table =
KnoxShellTable.builder().csv().withHeaders().string(csvString);
+ }
+ else {
+ table = KnoxShellTable.builder().csv().withHeaders().url(url);
+ }
}
else {
- table = KnoxShellTable.builder().csv().url(url);
+ if (url.startsWith("$")) {
+ // a knoxshell variable is a csv file as a string
+ String csvString = (String) getVariables().get(url.substring(1));
+ table = KnoxShellTable.builder().csv().string(csvString);
+ }
+ else {
+ table = KnoxShellTable.builder().csv().url(url);
+ }
}
} catch (IOException e) {
e.printStackTrace();
diff --git
a/gateway-shell/src/main/java/org/apache/knox/gateway/shell/commands/WebHDFSCommand.java
b/gateway-shell/src/main/java/org/apache/knox/gateway/shell/commands/WebHDFSCommand.java
new file mode 100644
index 0000000..4dafc27
--- /dev/null
+++
b/gateway-shell/src/main/java/org/apache/knox/gateway/shell/commands/WebHDFSCommand.java
@@ -0,0 +1,426 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.knox.gateway.shell.commands;
+
+import java.io.Console;
+import java.io.File;
+import java.io.IOException;
+import java.net.URISyntaxException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Calendar;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.TimeZone;
+
+import org.apache.knox.gateway.shell.CredentialCollectionException;
+import org.apache.knox.gateway.shell.CredentialCollector;
+import org.apache.knox.gateway.shell.KnoxSession;
+import org.apache.knox.gateway.shell.KnoxShellException;
+import org.apache.knox.gateway.shell.hdfs.Hdfs;
+import org.apache.knox.gateway.shell.hdfs.Status.Response;
+import org.apache.knox.gateway.shell.table.KnoxShellTable;
+import org.apache.knox.gateway.util.JsonUtils;
+import org.codehaus.groovy.tools.shell.Groovysh;
+
+public class WebHDFSCommand extends AbstractKnoxShellCommand {
+ private Map<String, KnoxSession> sessions = new HashMap<>();
+
+ public WebHDFSCommand(Groovysh shell) {
+ super(shell, ":filesystem", ":fs");
+ }
+
+ @Override
+ public String getUsage() {
+ String usage = "Usage: \n" +
+ " :fs ls {target-path} \n" +
+ " :fs cat {target-path} \n" +
+ " :fs get {from-path} {to-path} \n" +
+ " :fs put {from-path} {tp-path} \n" +
+ " :fs rm {target-path} \n" +
+ " :fs mkdir {dir-path} \n";
+ return usage;
+ }
+
+ @Override
+ public Object execute(List<String> args) {
+ Map<String, String> mounts = getMountPoints();
+ if (args.isEmpty()) {
+ args.add("ls");
+ }
+ if (args.get(0).equalsIgnoreCase("mount")) {
+ String url = args.get(1);
+ String mountPoint = args.get(2);
+ return mount(mounts, url, mountPoint);
+ }
+ else if (args.get(0).equalsIgnoreCase("unmount")) {
+ String mountPoint = args.get(1);
+ unmount(mounts, mountPoint);
+ }
+ else if (args.get(0).equalsIgnoreCase("mounts")) {
+ return listMounts(mounts);
+ }
+ else if (args.get(0).equalsIgnoreCase("ls")) {
+ String path = args.get(1);
+ return listStatus(mounts, path);
+ }
+ else if (args.get(0).equalsIgnoreCase("put")) {
+ // Hdfs.put( session ).file( dataFile ).to( dataDir + "/" + dataFile
).now()
+ // :fs put from-path to-path
+ String localFile = args.get(1);
+ String path = args.get(2);
+ int permission = 755;
+ if (args.size() >= 4) {
+ permission = Integer.parseInt(args.get(3));
+ }
+
+ return put(mounts, localFile, path, permission);
+ }
+ else if (args.get(0).equalsIgnoreCase("rm")) {
+ // Hdfs.rm( session ).file( dataFile ).now()
+ // :fs rm target-path
+ String path = args.get(1);
+ return remove(mounts, path);
+ }
+ else if (args.get(0).equalsIgnoreCase("cat")) {
+ // println Hdfs.get( session ).from( dataDir + "/" + dataFile
).now().string
+ // :fs cat target-path
+ String path = args.get(1);
+ return cat(mounts, path);
+ }
+ else if (args.get(0).equalsIgnoreCase("mkdir")) {
+ // println Hdfs.mkdir( session ).dir( directoryPath ).perm( "777"
).now().string
+ // :fs mkdir target-path [perms]
+ String path = args.get(1);
+ String perms = null;
+ if (args.size() == 3) {
+ perms = args.get(2);
+ }
+
+ return mkdir(mounts, path, perms);
+ }
+ else if (args.get(0).equalsIgnoreCase("get")) {
+ // println Hdfs.get( session ).from( dataDir + "/" + dataFile
).now().string
+ // :fs get from-path [to-path]
+ String path = args.get(1);
+
+ String mountPoint = determineMountPoint(path);
+ KnoxSession session = getSessionForMountPoint(mounts, mountPoint);
+ if (session != null) {
+ String from = determineTargetPath(path, mountPoint);
+ String to = null;
+ if (args.size() > 2) {
+ to = args.get(2);
+ }
+ else {
+ to = System.getProperty("user.home") + File.separator +
+ path.substring(path.lastIndexOf(File.separator));
+ }
+ return get(mountPoint, from, to);
+ }
+ else {
+ return "No session established for mountPoint: " + mountPoint + " Use
:fs mount {topology-url} {mountpoint-name}";
+ }
+ }
+ else {
+ System.out.println("Unknown filesystem command");
+ System.out.println(getUsage());
+ }
+ return "";
+ }
+
+ private String get(String mountPoint, String from, String to) {
+ String result = null;
+ try {
+ Hdfs.get(sessions.get(mountPoint)).from(from).file(to).now().getString();
+ result = "Successfully copied: " + from + " to: " + to;
+ } catch (KnoxShellException | IOException e) {
+ e.printStackTrace();
+ result = "Exception ocurred: " + e.getMessage();
+ }
+ return result;
+ }
+
+ private String mkdir(Map<String, String> mounts, String path, String perms) {
+ String result = null;
+ String mountPoint = determineMountPoint(path);
+ KnoxSession session = getSessionForMountPoint(mounts, mountPoint);
+ if (session != null) {
+ String targetPath = determineTargetPath(path, mountPoint);
+ if (!exists(session, targetPath)) {
+ try {
+ if (perms != null) {
+
Hdfs.mkdir(sessions.get(mountPoint)).dir(targetPath).now().getString();
+ }
+ else {
+ Hdfs.mkdir(session).dir(targetPath).perm(perms).now().getString();
+ }
+ result = "Successfully created directory: " + targetPath;
+ } catch (KnoxShellException | IOException e) {
+ e.printStackTrace();
+ result = "Exception ocurred: " + e.getMessage();
+ }
+ }
+ else {
+ result = targetPath + " already exists";
+ }
+ }
+ else {
+ result = "No session established for mountPoint: " + mountPoint + " Use
:fs mount {topology-url} {mountpoint-name}";
+ }
+ return result;
+ }
+
+ private String cat(Map<String, String> mounts, String path) {
+ String response = null;
+ String mountPoint = determineMountPoint(path);
+ KnoxSession session = getSessionForMountPoint(mounts, mountPoint);
+ if (session != null) {
+ String targetPath = determineTargetPath(path, mountPoint);
+ try {
+ String contents = Hdfs.get(session).from(targetPath).now().getString();
+ response = contents;
+ } catch (KnoxShellException | IOException e) {
+ e.printStackTrace();
+ response = "Exception ocurred: " + e.getMessage();
+ }
+ }
+ else {
+ response = "No session established for mountPoint: " + mountPoint + "
Use :fs mount {topology-url} {mountpoint-name}";
+ }
+ return response;
+ }
+
+ private String remove(Map<String, String> mounts, String path) {
+ String mountPoint = determineMountPoint(path);
+ KnoxSession session = getSessionForMountPoint(mounts, mountPoint);
+ if (session != null) {
+ String targetPath = determineTargetPath(path, mountPoint);
+ try {
+ Hdfs.rm(session).file(targetPath).now().getString();
+ } catch (KnoxShellException | IOException e) {
+ e.printStackTrace();
+ }
+ }
+ else {
+ return "No session established for mountPoint: " + mountPoint + " Use
:fs mount {topology-url} {mountpoint-name}";
+ }
+ return "Successfully removed: " + path;
+ }
+
+ private String put(Map<String, String> mounts, String localFile, String
path, int permission) {
+ String mountPoint = determineMountPoint(path);
+ KnoxSession session = getSessionForMountPoint(mounts, mountPoint);
+ if (session != null) {
+ String targetPath = determineTargetPath(path, mountPoint);
+ try {
+ boolean overwrite = false;
+ if (exists(session, targetPath)) {
+ if (collectClearInput(targetPath + " already exists would you like
to overwrite (Y/n)").equalsIgnoreCase("y")) {
+ overwrite = true;
+ }
+ }
+
Hdfs.put(session).file(localFile).to(targetPath).overwrite(overwrite).permission(permission).now().getString();
+ } catch (IOException e) {
+ e.printStackTrace();
+ return "Exception ocurred: " + e.getMessage();
+ }
+ }
+ else {
+ return "No session established for mountPoint: " + mountPoint + " Use
:fs mount {topology-url} {mountpoint-name}";
+ }
+ return "Successfully put: " + localFile + " to: " + path;
+ }
+
+ private boolean exists(KnoxSession session, String path) {
+ boolean rc = false;
+ try {
+ Response response = Hdfs.status(session).file(path).now();
+ rc = response.exists();
+ } catch (KnoxShellException e) {
+ // NOP
+ }
+ return rc;
+ }
+
+ private Object listStatus(Map<String, String> mounts, String path) {
+ Object response = null;
+ try {
+ String directory;
+ String mountPoint = determineMountPoint(path);
+ if (mountPoint != null) {
+ KnoxSession session = getSessionForMountPoint(mounts, mountPoint);
+ if (session != null) {
+ directory = determineTargetPath(path, mountPoint);
+ String json = Hdfs.ls(session).dir(directory).now().getString();
+ Map<String,HashMap<String, ArrayList<HashMap<String, String>>>> map =
+ JsonUtils.getFileStatusesAsMap(json);
+ if (map != null) {
+ ArrayList<HashMap<String, String>> list =
map.get("FileStatuses").get("FileStatus");
+ KnoxShellTable table = buildTableFromListStatus(directory, list);
+ response = table;
+ }
+ }
+ else {
+ response = "No session established for mountPoint: " + mountPoint +
" Use :fs mount {topology-url} {mountpoint-name}";
+ }
+ }
+ else {
+ response = "No mountpoint found. Use ':fs mount {topologyURL}
{mountpoint}'.";
+ }
+ } catch (KnoxShellException | IOException e) {
+ response = "Exception ocurred: " + e.getMessage();
+ e.printStackTrace();
+ }
+ return response;
+ }
+
+ private KnoxShellTable listMounts(Map<String, String> mounts) {
+ KnoxShellTable table = new KnoxShellTable();
+ table.header("Mount Point").header("Topology URL");
+ for (String mountPoint : mounts.keySet()) {
+ table.row().value(mountPoint).value(mounts.get(mountPoint));
+ }
+ return table;
+ }
+
+ private void unmount(Map<String, String> mounts, String mountPoint) {
+ sessions.remove(mountPoint);
+ mounts.remove(mountPoint);
+ KnoxSession.persistMountPoints(mounts);
+ }
+
+ private String mount(Map<String, String> mounts, String url, String
mountPoint) {
+ KnoxSession session = establishSession(mountPoint, url);
+ if (session != null) {
+ mounts.put(mountPoint, url);
+ KnoxSession.persistMountPoints(mounts);
+ return url + " mounted as " + mountPoint;
+ }
+ return "Failed to mount " + url + " as " + mountPoint;
+ }
+
+ private KnoxSession getSessionForMountPoint(Map<String, String> mounts,
String mountPoint) {
+ KnoxSession session = sessions.get(mountPoint);
+ if (session == null) {
+ String url = mounts.get(mountPoint);
+ if (url != null) {
+ session = establishSession(mountPoint, url);
+ }
+ }
+ return session;
+ }
+
+ private KnoxSession establishSession(String mountPoint, String url) {
+ CredentialCollector dlg;
+ try {
+ dlg = login();
+ } catch (CredentialCollectionException e) {
+ e.printStackTrace();
+ return null;
+ }
+ String username = dlg.name();
+ String password = new String(dlg.chars());
+ KnoxSession session = null;
+ try {
+ session = KnoxSession.login(url, username, password);
+ sessions.put(mountPoint, session);
+ } catch (URISyntaxException e) {
+ e.printStackTrace();
+ }
+ return session;
+ }
+
+ private String collectClearInput(String prompt) {
+ Console c = System.console();
+ if (c == null) {
+ System.err.println("No console.");
+ System.exit(1);
+ }
+
+ String value = c.readLine(prompt);
+
+ return value;
+ }
+
+ private String determineTargetPath(String path, String mountPoint) {
+ String directory = null;
+ if (path.startsWith("/")) {
+ directory = stripMountPoint(path, mountPoint);
+ }
+ return directory;
+ }
+
+ private String stripMountPoint(String path, String mountPoint) {
+ String newPath = path.replace("/" + mountPoint, "");
+ return newPath;
+ }
+
+ private String determineMountPoint(String path) {
+ String mountPoint = null;
+ if (path.startsWith("/")) {
+ // does the user supplied path starts at a root
+ // if so check for a mountPoint based on the first element of the path
+ String[] pathElements = path.split("/");
+ mountPoint = pathElements[1];
+ }
+ return mountPoint;
+ }
+
+ private KnoxShellTable buildTableFromListStatus(String directory,
List<HashMap<String, String>> list) {
+ Calendar cal = Calendar.getInstance(TimeZone.getDefault(),
Locale.getDefault());
+ KnoxShellTable table = new KnoxShellTable();
+ table.title(directory);
+ table.header("permission")
+ .header("owner")
+ .header("group")
+ .header("length")
+ .header("modtime")
+ .header("name");
+
+ for (Map<String, String> map : list) {
+ cal.setTimeInMillis(Long.parseLong(map.get("modificationTime")));
+ table.row()
+ .value(map.get("permission"))
+ .value(map.get("owner"))
+ .value(map.get("group"))
+ .value(map.get("length"))
+ .value(cal.getTime())
+ .value(map.get("pathSuffix"));
+ }
+
+ return table;
+ }
+
+ protected Map<String, String> getMountPoints() {
+ Map<String, String> mounts = null;
+ try {
+ mounts = KnoxSession.loadMountPoints();
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ return mounts;
+ }
+
+ public static void main(String[] args) {
+ WebHDFSCommand cmd = new WebHDFSCommand(new Groovysh());
+ cmd.execute(new ArrayList<>(Arrays.asList(args)));
+ }
+}
diff --git
a/gateway-shell/src/main/java/org/apache/knox/gateway/shell/hdfs/Ls.java
b/gateway-shell/src/main/java/org/apache/knox/gateway/shell/hdfs/Ls.java
index 88d8d86..a219cdd 100644
--- a/gateway-shell/src/main/java/org/apache/knox/gateway/shell/hdfs/Ls.java
+++ b/gateway-shell/src/main/java/org/apache/knox/gateway/shell/hdfs/Ls.java
@@ -26,7 +26,7 @@ import org.apache.http.client.utils.URIBuilder;
import java.util.concurrent.Callable;
-class Ls {
+public class Ls {
public static class Request extends AbstractRequest<Response> {
diff --git
a/gateway-shell/src/main/java/org/apache/knox/gateway/shell/hdfs/Mkdir.java
b/gateway-shell/src/main/java/org/apache/knox/gateway/shell/hdfs/Mkdir.java
index f927953..aca4479 100644
--- a/gateway-shell/src/main/java/org/apache/knox/gateway/shell/hdfs/Mkdir.java
+++ b/gateway-shell/src/main/java/org/apache/knox/gateway/shell/hdfs/Mkdir.java
@@ -26,7 +26,7 @@ import org.apache.http.client.utils.URIBuilder;
import java.util.concurrent.Callable;
-class Mkdir {
+public class Mkdir {
public static class Request extends AbstractRequest<Response> {
diff --git
a/gateway-shell/src/main/java/org/apache/knox/gateway/shell/hdfs/Put.java
b/gateway-shell/src/main/java/org/apache/knox/gateway/shell/hdfs/Put.java
index 9d6e74e..82293be 100644
--- a/gateway-shell/src/main/java/org/apache/knox/gateway/shell/hdfs/Put.java
+++ b/gateway-shell/src/main/java/org/apache/knox/gateway/shell/hdfs/Put.java
@@ -34,7 +34,7 @@ import org.apache.http.util.EntityUtils;
import java.io.File;
import java.util.concurrent.Callable;
-class Put {
+public class Put {
public static class Request extends AbstractRequest<Response> {
diff --git
a/gateway-shell/src/main/java/org/apache/knox/gateway/shell/hdfs/Rm.java
b/gateway-shell/src/main/java/org/apache/knox/gateway/shell/hdfs/Rm.java
index 52f3a88..ee7026a 100644
--- a/gateway-shell/src/main/java/org/apache/knox/gateway/shell/hdfs/Rm.java
+++ b/gateway-shell/src/main/java/org/apache/knox/gateway/shell/hdfs/Rm.java
@@ -27,7 +27,7 @@ import org.apache.http.client.utils.URIBuilder;
import java.io.IOException;
import java.util.concurrent.Callable;
-class Rm {
+public class Rm {
public static class Request extends AbstractRequest<Response> {
diff --git
a/gateway-shell/src/main/java/org/apache/knox/gateway/shell/table/CSVKnoxShellTableBuilder.java
b/gateway-shell/src/main/java/org/apache/knox/gateway/shell/table/CSVKnoxShellTableBuilder.java
index bda2ab2..d3ceedc 100644
---
a/gateway-shell/src/main/java/org/apache/knox/gateway/shell/table/CSVKnoxShellTableBuilder.java
+++
b/gateway-shell/src/main/java/org/apache/knox/gateway/shell/table/CSVKnoxShellTableBuilder.java
@@ -18,7 +18,9 @@
package org.apache.knox.gateway.shell.table;
import java.io.BufferedReader;
+import java.io.ByteArrayInputStream;
import java.io.IOException;
+import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.Reader;
import java.net.URL;
@@ -39,33 +41,45 @@ public class CSVKnoxShellTableBuilder extends
KnoxShellTableBuilder {
}
public KnoxShellTable url(String url) throws IOException {
- int rowIndex = 0;
URL urlToCsv = new URL(url);
URLConnection connection = urlToCsv.openConnection();
try (Reader urlConnectionStreamReader = new
InputStreamReader(connection.getInputStream(), StandardCharsets.UTF_8);
BufferedReader csvReader = new
BufferedReader(urlConnectionStreamReader);) {
- if (title != null) {
- this.table.title(title);
+ buildTableFromCSVReader(csvReader);
+ }
+ return this.table;
+ }
+
+ public KnoxShellTable string(String csvString) throws IOException {
+ try (InputStream is = new
ByteArrayInputStream(csvString.getBytes(StandardCharsets.UTF_8)); Reader
stringStreamReader = new InputStreamReader(is, StandardCharsets.UTF_8);
+ BufferedReader csvReader = new BufferedReader(stringStreamReader);) {
+ buildTableFromCSVReader(csvReader);
+ }
+ return this.table;
+ }
+
+ private void buildTableFromCSVReader(BufferedReader csvReader) throws
IOException {
+ int rowIndex = 0;
+ if (title != null) {
+ this.table.title(title);
+ }
+ String row = null;
+ while ((row = csvReader.readLine()) != null) {
+ boolean addingHeaders = (withHeaders && rowIndex == 0);
+ if (!addingHeaders) {
+ this.table.row();
}
- String row = null;
- while ((row = csvReader.readLine()) != null) {
- boolean addingHeaders = (withHeaders && rowIndex == 0);
- if (!addingHeaders) {
- this.table.row();
- }
- String[] data = row.split(",");
+ String[] data = row.split(",", -1);
- for (String value : data) {
- if (addingHeaders) {
- this.table.header(value);
- } else {
- this.table.value(value);
- }
+ for (String value : data) {
+ if (addingHeaders) {
+ this.table.header(value);
+ } else {
+ this.table.value(value);
}
- rowIndex++;
}
+ rowIndex++;
}
- return this.table;
}
}
diff --git
a/gateway-shell/src/main/java/org/apache/knox/gateway/shell/table/KnoxShellTable.java
b/gateway-shell/src/main/java/org/apache/knox/gateway/shell/table/KnoxShellTable.java
index 47439d4..4c5368a 100644
---
a/gateway-shell/src/main/java/org/apache/knox/gateway/shell/table/KnoxShellTable.java
+++
b/gateway-shell/src/main/java/org/apache/knox/gateway/shell/table/KnoxShellTable.java
@@ -137,6 +137,10 @@ public class KnoxShellTable {
double[] colArray = new double[col.size()];
Conversions conversionMethod = null;
for (int i = 0; i < col.size(); i++) {
+ Object v = col.get(i);
+ if (v instanceof String && ((String) v).trim().isEmpty()) {
+ col.set(i, "0");
+ }
if (i == 0) {
conversionMethod = getConversion(col.get(i));
}
@@ -435,6 +439,10 @@ public class KnoxShellTable {
}
public String toCSV() {
- return new KnoxShellTableRenderer(this).toCSV();
+ return toCSV(null);
+ }
+
+ public String toCSV(String filePath) {
+ return new KnoxShellTableRenderer(this).toCSV(filePath);
}
}
diff --git
a/gateway-shell/src/main/java/org/apache/knox/gateway/shell/table/KnoxShellTableFileUtils.java
b/gateway-shell/src/main/java/org/apache/knox/gateway/shell/table/KnoxShellTableFileUtils.java
new file mode 100644
index 0000000..5eb1c79
--- /dev/null
+++
b/gateway-shell/src/main/java/org/apache/knox/gateway/shell/table/KnoxShellTableFileUtils.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.knox.gateway.shell.table;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+
+public class KnoxShellTableFileUtils {
+ public static void persistToFile(String filePath, final String content)
throws IOException {
+ final Path jsonFilePath = Paths.get(filePath);
+ if (!Files.exists(jsonFilePath.getParent())) {
+ Files.createDirectories(jsonFilePath.getParent());
+ }
+ Files.deleteIfExists(jsonFilePath);
+ Files.createFile(jsonFilePath);
+ setPermissions(jsonFilePath);
+ Files.write(jsonFilePath, content.getBytes(StandardCharsets.UTF_8));
+ }
+
+ private static void setPermissions(Path path) throws IOException {
+ // clear all flags for everybody
+ path.toFile().setReadable(false, false);
+ path.toFile().setWritable(false, false);
+ path.toFile().setExecutable(false, false);
+ // allow owners to read/write
+ path.toFile().setReadable(true, true);
+ path.toFile().setWritable(true, true);
+ }
+}
diff --git
a/gateway-shell/src/main/java/org/apache/knox/gateway/shell/table/KnoxShellTableJSONSerializer.java
b/gateway-shell/src/main/java/org/apache/knox/gateway/shell/table/KnoxShellTableJSONSerializer.java
index 24850e2..bf10b28 100644
---
a/gateway-shell/src/main/java/org/apache/knox/gateway/shell/table/KnoxShellTableJSONSerializer.java
+++
b/gateway-shell/src/main/java/org/apache/knox/gateway/shell/table/KnoxShellTableJSONSerializer.java
@@ -18,10 +18,6 @@
package org.apache.knox.gateway.shell.table;
import java.io.IOException;
-import java.nio.charset.StandardCharsets;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.Locale;
@@ -87,27 +83,10 @@ class KnoxShellTableJSONSerializer {
} else {
jsonResult =
JsonUtils.renderAsJsonString(KnoxShellTableCallHistory.getInstance().getCallHistory(table.id),
null, JSON_DATE_FORMAT.get());
}
- final Path jsonFilePath = Paths.get(filePath);
- if (!Files.exists(jsonFilePath.getParent())) {
- Files.createDirectories(jsonFilePath.getParent());
- }
- Files.deleteIfExists(jsonFilePath);
- Files.createFile(jsonFilePath);
- setPermissions(jsonFilePath);
- Files.write(jsonFilePath, jsonResult.getBytes(StandardCharsets.UTF_8));
+ KnoxShellTableFileUtils.persistToFile(filePath, jsonResult);
return "Successfully saved into " + filePath;
} catch (IOException e) {
throw new KnoxShellException("Error while saving KnoxShellTable JSON
into " + filePath, e);
}
}
-
- private static void setPermissions(Path path) throws IOException {
- // clear all flags for everybody
- path.toFile().setReadable(false, false);
- path.toFile().setWritable(false, false);
- path.toFile().setExecutable(false, false);
- // allow owners to read/write
- path.toFile().setReadable(true, true);
- path.toFile().setWritable(true, true);
- }
}
diff --git
a/gateway-shell/src/main/java/org/apache/knox/gateway/shell/table/KnoxShellTableRenderer.java
b/gateway-shell/src/main/java/org/apache/knox/gateway/shell/table/KnoxShellTableRenderer.java
index b38cac8..38f5bd2 100644
---
a/gateway-shell/src/main/java/org/apache/knox/gateway/shell/table/KnoxShellTableRenderer.java
+++
b/gateway-shell/src/main/java/org/apache/knox/gateway/shell/table/KnoxShellTableRenderer.java
@@ -17,6 +17,7 @@
*/
package org.apache.knox.gateway.shell.table;
+import java.io.IOException;
import java.util.HashMap;
import java.util.List;
import java.util.Locale;
@@ -38,6 +39,10 @@ class KnoxShellTableRenderer {
}
String toCSV() {
+ return toCSV(null);
+ }
+
+ String toCSV(String filePath) {
final StringBuilder csv = new StringBuilder();
String header;
for (int i = 0; i < tableToRender.headers.size(); i++) {
@@ -59,8 +64,17 @@ class KnoxShellTableRenderer {
}
}
}
+ String content = csv.toString();
+ if (filePath != null) {
+ try {
+ KnoxShellTableFileUtils.persistToFile(filePath, content);
+ } catch (IOException e) {
+ System.out.println("Persistence of CSV file has failed. " +
e.getMessage());
+ e.printStackTrace();
+ }
+ }
- return csv.toString();
+ return content;
}
@Override
diff --git
a/gateway-shell/src/test/java/org/apache/knox/gateway/shell/table/KnoxShellTableTest.java
b/gateway-shell/src/test/java/org/apache/knox/gateway/shell/table/KnoxShellTableTest.java
index e65278a..ac2f802 100644
---
a/gateway-shell/src/test/java/org/apache/knox/gateway/shell/table/KnoxShellTableTest.java
+++
b/gateway-shell/src/test/java/org/apache/knox/gateway/shell/table/KnoxShellTableTest.java
@@ -191,6 +191,14 @@ public class KnoxShellTableTest {
}
@Test
+ public void testCSVStringToTable() throws IOException {
+ String initialString = "colA, colB, colC\nvalue1, value2. value3\nvalue4,
value5, value6";
+
+ KnoxShellTable table =
KnoxShellTable.builder().csv().withHeaders().string(initialString);
+ assertEquals(table.rows.size(), 2);
+ }
+
+ @Test
public void testToAndFromJSON() throws IOException {
KnoxShellTable table = new KnoxShellTable();
diff --git
a/gateway-util-common/src/main/java/org/apache/knox/gateway/util/JsonUtils.java
b/gateway-util-common/src/main/java/org/apache/knox/gateway/util/JsonUtils.java
index 4eabe2c..9d2e613 100644
---
a/gateway-util-common/src/main/java/org/apache/knox/gateway/util/JsonUtils.java
+++
b/gateway-util-common/src/main/java/org/apache/knox/gateway/util/JsonUtils.java
@@ -19,6 +19,7 @@ package org.apache.knox.gateway.util;
import java.io.IOException;
import java.text.DateFormat;
+import java.util.ArrayList;
import java.util.HashMap;
import java.util.Map;
@@ -97,4 +98,18 @@ public class JsonUtils {
}
return map;
}
+
+ public static Map<String,HashMap<String, ArrayList<HashMap<String,
String>>>> getFileStatusesAsMap(String json) {
+ Map<String,HashMap<String, ArrayList<HashMap<String, String>>>> map = null;
+ JsonFactory factory = new JsonFactory();
+ ObjectMapper mapper = new ObjectMapper(factory);
+ TypeReference<HashMap<String,HashMap<String, ArrayList<HashMap<String,
String>>>>> typeRef
+ = new TypeReference<HashMap<String,HashMap<String,
ArrayList<HashMap<String, String>>>>>() {};
+ try {
+ map = mapper.readValue(json, typeRef);
+ } catch (IOException e) {
+ //LOG.failedToGetMapFromJsonString( json, e );
+ }
+ return map;
+ }
}
diff --git
a/gateway-util-common/src/test/java/org/apache/knox/gateway/util/JsonUtilsTest.java
b/gateway-util-common/src/test/java/org/apache/knox/gateway/util/JsonUtilsTest.java
index 84fb7a1..05ae692 100644
---
a/gateway-util-common/src/test/java/org/apache/knox/gateway/util/JsonUtilsTest.java
+++
b/gateway-util-common/src/test/java/org/apache/knox/gateway/util/JsonUtilsTest.java
@@ -17,15 +17,16 @@
*/
package org.apache.knox.gateway.util;
+import java.util.ArrayList;
import java.util.HashMap;
import java.util.Map;
import org.junit.Test;
-
import static org.hamcrest.CoreMatchers.containsString;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
public class JsonUtilsTest {
private String expiresIn = "\"expires_in\":\"1364487943100\"";
@@ -54,4 +55,46 @@ public class JsonUtilsTest {
assertEquals("Bearer", map.get("token_type"));
assertEquals("1364487943100", map.get("expires_in"));
}
+
+ @Test
+ public void testFileStatusesAsMap() {
+ String json = "{\n" +
+ " \"FileStatuses\":{\n" +
+ " \"FileStatus\":[\n" +
+ " {\n" +
+ " \"accessTime\":0,\n" +
+ " \"blockSize\":0,\n" +
+ " \"childrenNum\":3,\n" +
+ " \"fileId\":16389,\n" +
+ " \"group\":\"supergroup\",\n" +
+ " \"length\":0,\n" +
+ " \"modificationTime\":1581578495905,\n" +
+ " \"owner\":\"hdfs\",\n" +
+ " \"pathSuffix\":\"tmp\",\n" +
+ " \"permission\":\"1777\",\n" +
+ " \"replication\":0,\n" +
+ " \"storagePolicy\":0,\n" +
+ " \"type\":\"DIRECTORY\"\n" +
+ " },\n" +
+ " {\n" +
+ " \"accessTime\":0,\n" +
+ " \"blockSize\":0,\n" +
+ " \"childrenNum\":658,\n" +
+ " \"fileId\":16386,\n" +
+ " \"group\":\"supergroup\",\n" +
+ " \"length\":0,\n" +
+ " \"modificationTime\":1581578527580,\n" +
+ " \"owner\":\"hdfs\",\n" +
+ " \"pathSuffix\":\"user\",\n" +
+ " \"permission\":\"755\",\n" +
+ " \"replication\":0,\n" +
+ " \"storagePolicy\":0,\n" +
+ " \"type\":\"DIRECTORY\"\n" +
+ " } \n" +
+ " ]\n" +
+ " }\n" +
+ "}";
+ Map<String,HashMap<String, ArrayList<HashMap<String, String>>>> map =
JsonUtils.getFileStatusesAsMap(json);
+ assertNotNull(map);
+ }
}
