This is an automated email from the ASF dual-hosted git repository.
szita pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 0a95b3a HIVE-22490: Adding jars with special characters in their path
throws error (Adam Szita, reviewed by Peter Vary)
0a95b3a is described below
commit 0a95b3a057a424d22d5943295da6220e57c66b6e
Author: Adam Szita <sz...@cloudera.com>
AuthorDate: Fri Nov 29 15:23:03 2019 +0100
HIVE-22490: Adding jars with special characters in their path throws error
(Adam Szita, reviewed by Peter Vary)
---
.../hadoop/hive/ql/session/SessionState.java | 2 +-
.../hadoop/hive/ql/util/ResourceDownloader.java | 4 +-
.../hadoop/hive/ql/session/TestAddResource.java | 71 +++++++++++++---------
3 files changed, 45 insertions(+), 32 deletions(-)
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
index e224f2c..de6aebb 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
@@ -1505,7 +1505,7 @@ public class SessionState {
Set<String> downloadedValues = new HashSet<String>();
for (URI uri : downloadedURLs) {
- String resourceValue = uri.toString();
+ String resourceValue = uri.getPath();
downloadedValues.add(resourceValue);
localized.add(resourceValue);
if (reverseResourcePathMap.containsKey(resourceValue)) {
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/util/ResourceDownloader.java
b/ql/src/java/org/apache/hadoop/hive/ql/util/ResourceDownloader.java
index 5c5aae5..07dfc05 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/util/ResourceDownloader.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/util/ResourceDownloader.java
@@ -98,13 +98,13 @@ public class ResourceDownloader {
LOG.debug("Converting to local {}", srcUri);
File destinationDir = (subDir == null) ? resourceDir : new
File(resourceDir, subDir);
ensureDirectory(destinationDir);
- File destinationFile = new File(destinationDir, new
Path(srcUri.toString()).getName());
+ File destinationFile = new File(destinationDir, new
Path(srcUri).getName());
String dest = destinationFile.getCanonicalPath();
if (destinationFile.exists()) {
return dest;
}
FileSystem fs = FileSystem.get(srcUri, conf);
- fs.copyToLocalFile(new Path(srcUri.toString()), new Path(dest));
+ fs.copyToLocalFile(new Path(srcUri), new Path(dest));
// add "execute" permission to downloaded resource file (needed when
loading dll file)
FileUtil.chmod(dest, "ugo+rx", true);
return dest;
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/session/TestAddResource.java
b/ql/src/test/org/apache/hadoop/hive/ql/session/TestAddResource.java
index dabec44..cc25178 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/session/TestAddResource.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/session/TestAddResource.java
@@ -18,9 +18,9 @@
package org.apache.hadoop.hive.ql.session;
-import static org.junit.Assert.assertEquals;
-
+import java.io.BufferedWriter;
import java.io.File;
+import java.io.FileWriter;
import java.io.IOException;
import java.io.Writer;
import java.net.URI;
@@ -31,23 +31,19 @@ import java.util.LinkedList;
import java.util.List;
import java.util.Set;
-import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.conf.HiveConf;
-import org.junit.After;
+import org.apache.hadoop.hive.ql.session.SessionState.ResourceType;
+
import org.junit.Before;
import org.junit.Test;
import org.mockito.Mockito;
-import org.apache.hadoop.hive.ql.session.SessionState.ResourceType;
-import org.apache.hadoop.hive.ql.session.SessionState;
-import org.apache.hadoop.util.Shell;
-import java.io.BufferedWriter;
-import java.io.FileWriter;
+import static org.junit.Assert.assertEquals;
public class TestAddResource {
- private static final String TEST_JAR_DIR =
System.getProperty("test.tmp.dir", ".") + File.pathSeparator;
+ private static final String TEST_JAR_DIR =
System.getProperty("test.tmp.dir", ".") + File.separator;
private HiveConf conf;
private ResourceType t;
@@ -58,13 +54,43 @@ public class TestAddResource {
//Generate test jar files
for (int i = 1; i <= 5; i++) {
- Writer output = null;
- String dataFile = TEST_JAR_DIR + "testjar" + i + ".jar";
- File file = new File(dataFile);
- output = new BufferedWriter(new FileWriter(file));
- output.write("sample");
- output.close();
+ writeTestJarFile(TEST_JAR_DIR + "testjar" + i + ".jar", "sample");
+ }
+ }
+
+ private static void writeTestJarFile(String dataFile, String content) throws
IOException {
+ File file = new File(dataFile);
+ file.deleteOnExit();
+ Writer output = new BufferedWriter(new FileWriter(file));
+ output.write(content);
+ output.close();
+ }
+
+ /**
+ * Tests adding jar with special chars in the path:
+ * - works if it's given in URL encoded format
+ * - fails if it's not encoded (user should have done it).
+ * @throws Exception
+ */
+ @Test
+ public void testSpecialCharsInPath() throws Exception {
+ String filePath = TEST_JAR_DIR + "testjar-[specialchars].jar";
+ String filePathEncoded = TEST_JAR_DIR + "testjar-%5Bspecialchars%5D.jar";
+ writeTestJarFile(filePath, "sample");
+
+ SessionState ss = Mockito.spy(SessionState.start(conf).get());
+ try {
+ ss.add_resource(t, filePath);
+ } catch (RuntimeException e) {
+ assertEquals(URISyntaxException.class, e.getCause().getClass());
}
+ ss.add_resource(t, filePathEncoded);
+
+ Set<String> result = ss.list_resource(t, null);
+ assertEquals(1, result.size());
+ assertEquals(filePath, result.stream().findFirst().get());
+
+ ss.close();
}
// Check that all the jars are added to the classpath
@@ -321,19 +347,6 @@ public class TestAddResource {
ss.close();
}
- @After
- public void tearDown() {
- // delete sample jars
- for (int i = 1; i <= 5; i++) {
- String dataFile = TEST_JAR_DIR + "testjar" + i + ".jar";
-
- File f = new File(dataFile);
- if (!f.delete()) {
- throw new RuntimeException("Could not delete the data file");
- }
- }
- }
-
private <T> List<T> union(List<T> list1, List<T> list2) {
Set<T> set = new HashSet<T>();
