This is an automated email from the ASF dual-hosted git repository.
stevel pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.3 by this push:
new 09e8e5c HADOOP-17870. Http Filesystem to qualify relative paths.
(#3338)
09e8e5c is described below
commit 09e8e5c5cb74f151a4a7ab314b7cb2741372bcad
Author: Yellow Flash <[email protected]>
AuthorDate: Tue Aug 31 18:25:52 2021 +0530
HADOOP-17870. Http Filesystem to qualify relative paths. (#3338)
Contributed by Yellowflash
Change-Id: I217da06a1a2e5c0ca2b324f8e21baa0846f64858
---
.../hadoop/fs/http/AbstractHttpFileSystem.java | 5 ++-
.../apache/hadoop/fs/http/TestHttpFileSystem.java | 44 ++++++++++++++++------
2 files changed, 36 insertions(+), 13 deletions(-)
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/http/AbstractHttpFileSystem.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/http/AbstractHttpFileSystem.java
index baf0a81..44577ba 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/http/AbstractHttpFileSystem.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/http/AbstractHttpFileSystem.java
@@ -60,7 +60,8 @@ abstract class AbstractHttpFileSystem extends FileSystem {
@Override
public FSDataInputStream open(Path path, int bufferSize) throws IOException {
- URLConnection conn = path.toUri().toURL().openConnection();
+ URI pathUri = makeQualified(path).toUri();
+ URLConnection conn = pathUri.toURL().openConnection();
InputStream in = conn.getInputStream();
return new FSDataInputStream(new HttpDataInputStream(in));
}
@@ -111,7 +112,7 @@ abstract class AbstractHttpFileSystem extends FileSystem {
@Override
public FileStatus getFileStatus(Path path) throws IOException {
- return new FileStatus(-1, false, 1, DEFAULT_BLOCK_SIZE, 0, path);
+ return new FileStatus(-1, false, 1, DEFAULT_BLOCK_SIZE, 0,
makeQualified(path));
}
/**
diff --git
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/http/TestHttpFileSystem.java
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/http/TestHttpFileSystem.java
index 0902c04..51692ed 100644
---
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/http/TestHttpFileSystem.java
+++
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/http/TestHttpFileSystem.java
@@ -26,6 +26,7 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;
import org.junit.Test;
+import org.junit.jupiter.api.BeforeEach;
import java.io.IOException;
import java.io.InputStream;
@@ -33,6 +34,7 @@ import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
import java.nio.charset.StandardCharsets;
+import java.util.stream.IntStream;
import static org.junit.Assert.assertEquals;
@@ -40,28 +42,48 @@ import static org.junit.Assert.assertEquals;
* Testing HttpFileSystem.
*/
public class TestHttpFileSystem {
+ private final Configuration conf = new Configuration(false);
+
+ @BeforeEach
+ public void setUp() {
+ conf.set("fs.http.impl", HttpFileSystem.class.getCanonicalName());
+ }
+
@Test
public void testHttpFileSystem() throws IOException, URISyntaxException,
InterruptedException {
- Configuration conf = new Configuration(false);
- conf.set("fs.http.impl", HttpFileSystem.class.getCanonicalName());
final String data = "foo";
-
try (MockWebServer server = new MockWebServer()) {
- server.enqueue(new MockResponse().setBody(data));
+ IntStream.rangeClosed(1, 3).forEach(i -> server.enqueue(new
MockResponse().setBody(data)));
server.start();
URI uri = URI.create(String.format("http://%s:%d";, server.getHostName(),
server.getPort()));
FileSystem fs = FileSystem.get(uri, conf);
- try (InputStream is = fs.open(
- new Path(new URL(uri.toURL(), "/foo").toURI()),
- 4096)) {
- byte[] buf = new byte[data.length()];
- IOUtils.readFully(is, buf, 0, buf.length);
- assertEquals(data, new String(buf, StandardCharsets.UTF_8));
- }
+ assertSameData(fs, new Path(new URL(uri.toURL(), "/foo").toURI()), data);
+ assertSameData(fs, new Path("/foo"), data);
+ assertSameData(fs, new Path("foo"), data);
RecordedRequest req = server.takeRequest();
assertEquals("/foo", req.getPath());
}
}
+
+ @Test
+ public void testHttpFileStatus() throws IOException, URISyntaxException,
InterruptedException {
+ URI uri = new URI("http://www.example.com";);
+ FileSystem fs = FileSystem.get(uri, conf);
+ URI expectedUri = uri.resolve("/foo");
+ assertEquals(fs.getFileStatus(new Path(new Path(uri),
"/foo")).getPath().toUri(), expectedUri);
+ assertEquals(fs.getFileStatus(new Path("/foo")).getPath().toUri(),
expectedUri);
+ assertEquals(fs.getFileStatus(new Path("foo")).getPath().toUri(),
expectedUri);
+ }
+
+ private void assertSameData(FileSystem fs, Path path, String data) throws
IOException {
+ try (InputStream is = fs.open(
+ path,
+ 4096)) {
+ byte[] buf = new byte[data.length()];
+ IOUtils.readFully(is, buf, 0, buf.length);
+ assertEquals(data, new String(buf, StandardCharsets.UTF_8));
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
