[
https://issues.apache.org/jira/browse/ARROW-2025?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16338562#comment-16338562
]
ASF GitHub Bot commented on ARROW-2025:
---------------------------------------
wesm closed pull request #1499: ARROW-2025: [C++] Creating multiple equivalent
`HadoopFileSystem`s works fine
URL: https://github.com/apache/arrow/pull/1499
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/cpp/src/arrow/io/hdfs-internal.cc
b/cpp/src/arrow/io/hdfs-internal.cc
index 545b2d17d..efceb8ae6 100644
--- a/cpp/src/arrow/io/hdfs-internal.cc
+++ b/cpp/src/arrow/io/hdfs-internal.cc
@@ -310,6 +310,10 @@ void
LibHdfsShim::BuilderSetKerbTicketCachePath(hdfsBuilder* bld,
this->hdfsBuilderSetKerbTicketCachePath(bld, kerbTicketCachePath);
}
+void LibHdfsShim::BuilderSetForceNewInstance(hdfsBuilder* bld) {
+ this->hdfsBuilderSetForceNewInstance(bld);
+}
+
hdfsFS LibHdfsShim::BuilderConnect(hdfsBuilder* bld) {
return this->hdfsBuilderConnect(bld);
}
@@ -490,6 +494,7 @@ Status LibHdfsShim::GetRequiredSymbols() {
GET_SYMBOL_REQUIRED(this, hdfsBuilderSetNameNodePort);
GET_SYMBOL_REQUIRED(this, hdfsBuilderSetUserName);
GET_SYMBOL_REQUIRED(this, hdfsBuilderSetKerbTicketCachePath);
+ GET_SYMBOL_REQUIRED(this, hdfsBuilderSetForceNewInstance);
GET_SYMBOL_REQUIRED(this, hdfsBuilderConnect);
GET_SYMBOL_REQUIRED(this, hdfsCreateDirectory);
GET_SYMBOL_REQUIRED(this, hdfsDelete);
diff --git a/cpp/src/arrow/io/hdfs-internal.h b/cpp/src/arrow/io/hdfs-internal.h
index df925cf62..f0fce23c2 100644
--- a/cpp/src/arrow/io/hdfs-internal.h
+++ b/cpp/src/arrow/io/hdfs-internal.h
@@ -51,6 +51,7 @@ struct LibHdfsShim {
void (*hdfsBuilderSetUserName)(hdfsBuilder* bld, const char* userName);
void (*hdfsBuilderSetKerbTicketCachePath)(hdfsBuilder* bld,
const char* kerbTicketCachePath);
+ void (*hdfsBuilderSetForceNewInstance)(hdfsBuilder* bld);
hdfsFS (*hdfsBuilderConnect)(hdfsBuilder* bld);
int (*hdfsDisconnect)(hdfsFS fs);
@@ -95,6 +96,7 @@ struct LibHdfsShim {
this->hdfsBuilderSetNameNodePort = nullptr;
this->hdfsBuilderSetUserName = nullptr;
this->hdfsBuilderSetKerbTicketCachePath = nullptr;
+ this->hdfsBuilderSetForceNewInstance = nullptr;
this->hdfsBuilderConnect = nullptr;
this->hdfsDisconnect = nullptr;
this->hdfsOpenFile = nullptr;
@@ -138,6 +140,8 @@ struct LibHdfsShim {
void BuilderSetKerbTicketCachePath(hdfsBuilder* bld, const char*
kerbTicketCachePath);
+ void BuilderSetForceNewInstance(hdfsBuilder* bld);
+
hdfsFS BuilderConnect(hdfsBuilder* bld);
int Disconnect(hdfsFS fs);
diff --git a/cpp/src/arrow/io/hdfs.cc b/cpp/src/arrow/io/hdfs.cc
index 6e3e4a7a1..6c569ae1e 100644
--- a/cpp/src/arrow/io/hdfs.cc
+++ b/cpp/src/arrow/io/hdfs.cc
@@ -335,6 +335,7 @@ class HadoopFileSystem::HadoopFileSystemImpl {
if (!config->kerb_ticket.empty()) {
driver_->BuilderSetKerbTicketCachePath(builder,
config->kerb_ticket.c_str());
}
+ driver_->BuilderSetForceNewInstance(builder);
fs_ = driver_->BuilderConnect(builder);
if (fs_ == nullptr) {
diff --git a/cpp/src/arrow/io/io-hdfs-test.cc b/cpp/src/arrow/io/io-hdfs-test.cc
index 5305b4774..f2ded6ff4 100644
--- a/cpp/src/arrow/io/io-hdfs-test.cc
+++ b/cpp/src/arrow/io/io-hdfs-test.cc
@@ -178,6 +178,21 @@ TYPED_TEST(TestHadoopFileSystem, ConnectsAgain) {
ASSERT_OK(client->Disconnect());
}
+TYPED_TEST(TestHadoopFileSystem, MultipleClients) {
+ SKIP_IF_NO_DRIVER();
+
+ std::shared_ptr<HadoopFileSystem> client1;
+ std::shared_ptr<HadoopFileSystem> client2;
+ ASSERT_OK(HadoopFileSystem::Connect(&this->conf_, &client1));
+ ASSERT_OK(HadoopFileSystem::Connect(&this->conf_, &client2));
+ ASSERT_OK(client1->Disconnect());
+
+ // client2 continues to function after equivalent client1 has shutdown
+ std::vector<HdfsPathInfo> listing;
+ EXPECT_OK(client2->ListDirectory(this->scratch_dir_, &listing));
+ ASSERT_OK(client2->Disconnect());
+}
+
TYPED_TEST(TestHadoopFileSystem, MakeDirectory) {
SKIP_IF_NO_DRIVER();
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
> [Python/C++] HDFS Client disconnect closes all open clients
> -----------------------------------------------------------
>
> Key: ARROW-2025
> URL: https://issues.apache.org/jira/browse/ARROW-2025
> Project: Apache Arrow
> Issue Type: Bug
> Components: C++, Python
> Reporter: Jim Crist
> Assignee: Jim Crist
> Priority: Major
> Labels: pull-request-available
> Fix For: 0.9.0
>
>
> In the python library, if an instance of `HadoopFileSystem` is garbage
> collected, all other existing instances become invalid. I haven't checked
> with a C++ only example, but from reading the cython code I can't see how
> cython is responsible, so I think this is a bug in the C++ library.
>
> {code:java}
> >>> import pyarrow as pa
> >>> h = pa.hdfs.connect()
> 18/01/24 16:54:25 WARN util.NativeCodeLoader: Unable to load native-hadoop
> library for your platform... using builtin-java classes where applicable
> 18/01/24 16:54:26 WARN shortcircuit.DomainSocketFactory: The short-circuit
> local reads feature cannot be used because libhadoop cannot be loaded.
> >>> h.ls("/")
> ['/benchmarks', '/hbase', '/tmp', '/user', '/var']
> >>> h2 = pa.hdfs.connect()
> >>> del h # close one client
> >>> h2.ls("/") # all filesystem operations now fail
> hdfsListDirectory(/): FileSystem#listStatus error:
> IOException: Filesystem closedjava.io.IOException: Filesystem closed
> at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:865)
> at org.apache.hadoop.hdfs.DFSClient.listPaths(DFSClient.java:2106)
> at org.apache.hadoop.hdfs.DFSClient.listPaths(DFSClient.java:2092)
> at
> org.apache.hadoop.hdfs.DistributedFileSystem.listStatusInternal(DistributedFileSystem.java:743)
> at
> org.apache.hadoop.hdfs.DistributedFileSystem.access$600(DistributedFileSystem.java:113)
> at
> org.apache.hadoop.hdfs.DistributedFileSystem$16.doCall(DistributedFileSystem.java:808)
> at
> org.apache.hadoop.hdfs.DistributedFileSystem$16.doCall(DistributedFileSystem.java:804)
> at
> org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81)
> at
> org.apache.hadoop.hdfs.DistributedFileSystem.listStatus(DistributedFileSystem.java:804)
> Traceback (most recent call last):
> File "<stdin>", line 1, in <module>
> File "/opt/conda/lib/python3.6/site-packages/pyarrow/hdfs.py", line 88, in
> ls
> return super(HadoopFileSystem, self).ls(path, detail)
> File "io-hdfs.pxi", line 248, in pyarrow.lib.HadoopFileSystem.ls
> File "error.pxi", line 79, in pyarrow.lib.check_status
> pyarrow.lib.ArrowIOError: HDFS: list directory failed
> >>> h2.is_open # The python object still thinks it's open
> True
> {code}
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)