This is an automated email from the ASF dual-hosted git repository.
felipecrv pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/main by this push:
new e5145bff90 GH-39339: [C++] Add
ForceCachedHierarchicalNamespaceSupport to help with testing (#39340)
e5145bff90 is described below
commit e5145bff901778360f6faba3be27efa3d9522976
Author: Felipe Oliveira Carvalho <[email protected]>
AuthorDate: Thu Dec 21 15:00:22 2023 -0300
GH-39339: [C++] Add ForceCachedHierarchicalNamespaceSupport to help with
testing (#39340)
### Rationale for this change
This ensures all the branches in the `AzureFileSystem` code operations are
tested.
For instance, many operations executed on a missing container, wouldn't
get a `HNSSupport::kContainerNotFound` error if the cached `HNSSupport` was
already known due to a previous operation that cached the `HNSSupport`
value.
### What changes are included in this PR?
Introduction of the helper that overrides `cached_hns_support_` and
enumeration of the scenarios.
### Are these changes tested?
Yes. This is a test improvement PR.
* Closes: #39339
Authored-by: Felipe Oliveira Carvalho <[email protected]>
Signed-off-by: Felipe Oliveira Carvalho <[email protected]>
---
cpp/src/arrow/filesystem/azurefs.cc | 36 ++-
cpp/src/arrow/filesystem/azurefs.h | 5 +
cpp/src/arrow/filesystem/azurefs_test.cc | 453 +++++++++++++++++--------------
3 files changed, 291 insertions(+), 203 deletions(-)
diff --git a/cpp/src/arrow/filesystem/azurefs.cc
b/cpp/src/arrow/filesystem/azurefs.cc
index d72ead92ed..27bdb5092a 100644
--- a/cpp/src/arrow/filesystem/azurefs.cc
+++ b/cpp/src/arrow/filesystem/azurefs.cc
@@ -941,14 +941,38 @@ class AzureFileSystem::Impl {
break;
}
ARROW_ASSIGN_OR_RAISE(
- cached_hns_support_,
+ auto hns_support,
internal::CheckIfHierarchicalNamespaceIsEnabled(adlfs_client,
options_));
- DCHECK_NE(cached_hns_support_, HNSSupport::kUnknown);
- // Caller should handle kContainerNotFound case appropriately.
- return cached_hns_support_;
+ DCHECK_NE(hns_support, HNSSupport::kUnknown);
+ if (hns_support == HNSSupport::kContainerNotFound) {
+ // Caller should handle kContainerNotFound case appropriately as it
knows the
+ // container this refers to, but the cached value in that case should
remain
+ // kUnknown before we get a CheckIfHierarchicalNamespaceIsEnabled result
that
+ // is not kContainerNotFound.
+ cached_hns_support_ = HNSSupport::kUnknown;
+ } else {
+ cached_hns_support_ = hns_support;
+ }
+ return hns_support;
}
public:
+ /// This is used from unit tests to ensure we perform operations on all the
+ /// possible states of cached_hns_support_.
+ void ForceCachedHierarchicalNamespaceSupport(int support) {
+ auto hns_support = static_cast<HNSSupport>(support);
+ switch (hns_support) {
+ case HNSSupport::kUnknown:
+ case HNSSupport::kContainerNotFound:
+ case HNSSupport::kDisabled:
+ case HNSSupport::kEnabled:
+ cached_hns_support_ = hns_support;
+ return;
+ }
+ // This is reachable if an invalid int is cast to enum class HNSSupport.
+ DCHECK(false) << "Invalid enum HierarchicalNamespaceSupport value.";
+ }
+
Result<FileInfo> GetFileInfo(const AzureLocation& location) {
if (location.container.empty()) {
DCHECK(location.path.empty());
@@ -1560,6 +1584,10 @@ AzureFileSystem::AzureFileSystem(std::unique_ptr<Impl>&&
impl)
default_async_is_sync_ = false;
}
+void AzureFileSystem::ForceCachedHierarchicalNamespaceSupport(int hns_support)
{
+ impl_->ForceCachedHierarchicalNamespaceSupport(hns_support);
+}
+
Result<std::shared_ptr<AzureFileSystem>> AzureFileSystem::Make(
const AzureOptions& options, const io::IOContext& io_context) {
ARROW_ASSIGN_OR_RAISE(auto impl, AzureFileSystem::Impl::Make(options,
io_context));
diff --git a/cpp/src/arrow/filesystem/azurefs.h
b/cpp/src/arrow/filesystem/azurefs.h
index be3ca5ba23..69f6295237 100644
--- a/cpp/src/arrow/filesystem/azurefs.h
+++ b/cpp/src/arrow/filesystem/azurefs.h
@@ -44,6 +44,8 @@ class DataLakeServiceClient;
namespace arrow::fs {
+class TestAzureFileSystem;
+
/// Options for the AzureFileSystem implementation.
struct ARROW_EXPORT AzureOptions {
/// \brief hostname[:port] of the Azure Blob Storage Service.
@@ -156,6 +158,9 @@ class ARROW_EXPORT AzureFileSystem : public FileSystem {
explicit AzureFileSystem(std::unique_ptr<Impl>&& impl);
+ friend class TestAzureFileSystem;
+ void ForceCachedHierarchicalNamespaceSupport(int hns_support);
+
public:
~AzureFileSystem() override = default;
diff --git a/cpp/src/arrow/filesystem/azurefs_test.cc
b/cpp/src/arrow/filesystem/azurefs_test.cc
index ecf7522b98..3266c1bfda 100644
--- a/cpp/src/arrow/filesystem/azurefs_test.cc
+++ b/cpp/src/arrow/filesystem/azurefs_test.cc
@@ -62,7 +62,6 @@ namespace arrow {
using internal::TemporaryDir;
namespace fs {
using internal::ConcatAbstractPath;
-namespace {
namespace bp = boost::process;
using ::testing::IsEmpty;
@@ -354,7 +353,7 @@ class TestAzureFileSystem : public ::testing::Test {
bool set_up_succeeded_ = false;
AzureOptions options_;
- std::shared_ptr<FileSystem> fs_;
+ std::shared_ptr<AzureFileSystem> fs_dont_use_directly_; // use fs()
std::unique_ptr<Blobs::BlobServiceClient> blob_service_client_;
std::unique_ptr<DataLake::DataLakeServiceClient> datalake_service_client_;
@@ -362,6 +361,18 @@ class TestAzureFileSystem : public ::testing::Test {
TestAzureFileSystem() : rng_(std::random_device()()) {}
virtual Result<BaseAzureEnv*> GetAzureEnv() const = 0;
+ virtual HNSSupport CachedHNSSupport(const BaseAzureEnv& env) const = 0;
+
+ FileSystem* fs(HNSSupport cached_hns_support) const {
+ auto* fs_ptr = fs_dont_use_directly_.get();
+
fs_ptr->ForceCachedHierarchicalNamespaceSupport(static_cast<int>(cached_hns_support));
+ return fs_ptr;
+ }
+
+ FileSystem* fs() const {
+ EXPECT_OK_AND_ASSIGN(auto env, GetAzureEnv());
+ return fs(CachedHNSSupport(*env));
+ }
static Result<AzureOptions> MakeOptions(BaseAzureEnv* env) {
AzureOptions options;
@@ -395,7 +406,7 @@ class TestAzureFileSystem : public ::testing::Test {
EXPECT_OK_AND_ASSIGN(options_, options_res);
}
- ASSERT_OK_AND_ASSIGN(fs_, AzureFileSystem::Make(options_));
+ ASSERT_OK_AND_ASSIGN(fs_dont_use_directly_,
AzureFileSystem::Make(options_));
EXPECT_OK_AND_ASSIGN(blob_service_client_,
options_.MakeBlobServiceClient());
EXPECT_OK_AND_ASSIGN(datalake_service_client_,
options_.MakeDataLakeServiceClient());
set_up_succeeded_ = true;
@@ -435,7 +446,7 @@ class TestAzureFileSystem : public ::testing::Test {
void UploadLines(const std::vector<std::string>& lines, const std::string&
path,
int total_size) {
- ASSERT_OK_AND_ASSIGN(auto output, fs_->OpenOutputStream(path, {}));
+ ASSERT_OK_AND_ASSIGN(auto output, fs()->OpenOutputStream(path, {}));
const auto all_lines = std::accumulate(lines.begin(), lines.end(),
std::string(""));
ASSERT_OK(output->Write(all_lines));
ASSERT_OK(output->Close());
@@ -461,19 +472,19 @@ class TestAzureFileSystem : public ::testing::Test {
const auto sub_directory_path = ConcatAbstractPath(directory_path,
"new-sub");
const auto sub_blob_path = ConcatAbstractPath(sub_directory_path,
"sub.txt");
const auto top_blob_path = ConcatAbstractPath(directory_path, "top.txt");
- ASSERT_OK(fs_->CreateDir(sub_directory_path, true));
- ASSERT_OK_AND_ASSIGN(auto output, fs_->OpenOutputStream(sub_blob_path));
+ ASSERT_OK(fs()->CreateDir(sub_directory_path, true));
+ ASSERT_OK_AND_ASSIGN(auto output, fs()->OpenOutputStream(sub_blob_path));
ASSERT_OK(output->Write(std::string_view("sub")));
ASSERT_OK(output->Close());
- ASSERT_OK_AND_ASSIGN(output, fs_->OpenOutputStream(top_blob_path));
+ ASSERT_OK_AND_ASSIGN(output, fs()->OpenOutputStream(top_blob_path));
ASSERT_OK(output->Write(std::string_view("top")));
ASSERT_OK(output->Close());
- AssertFileInfo(fs_.get(), data.container_name, FileType::Directory);
- AssertFileInfo(fs_.get(), directory_path, FileType::Directory);
- AssertFileInfo(fs_.get(), sub_directory_path, FileType::Directory);
- AssertFileInfo(fs_.get(), sub_blob_path, FileType::File);
- AssertFileInfo(fs_.get(), top_blob_path, FileType::File);
+ AssertFileInfo(fs(), data.container_name, FileType::Directory);
+ AssertFileInfo(fs(), directory_path, FileType::Directory);
+ AssertFileInfo(fs(), sub_directory_path, FileType::Directory);
+ AssertFileInfo(fs(), sub_blob_path, FileType::File);
+ AssertFileInfo(fs(), top_blob_path, FileType::File);
paths->container = data.container_name;
paths->directory = directory_path;
@@ -538,52 +549,52 @@ class TestAzureFileSystem : public ::testing::Test {
const auto directory_path = data.RandomDirectoryPath(rng_);
if (WithHierarchicalNamespace()) {
- ASSERT_OK(fs_->CreateDir(directory_path, true));
- arrow::fs::AssertFileInfo(fs_.get(), directory_path,
FileType::Directory);
- ASSERT_OK(fs_->DeleteDir(directory_path));
- arrow::fs::AssertFileInfo(fs_.get(), directory_path, FileType::NotFound);
+ ASSERT_OK(fs()->CreateDir(directory_path, true));
+ AssertFileInfo(fs(), directory_path, FileType::Directory);
+ ASSERT_OK(fs()->DeleteDir(directory_path));
+ AssertFileInfo(fs(), directory_path, FileType::NotFound);
} else {
// There is only virtual directory without hierarchical namespace
// support. So the CreateDir() and DeleteDir() do nothing.
- ASSERT_OK(fs_->CreateDir(directory_path));
- arrow::fs::AssertFileInfo(fs_.get(), directory_path, FileType::NotFound);
- ASSERT_OK(fs_->DeleteDir(directory_path));
- arrow::fs::AssertFileInfo(fs_.get(), directory_path, FileType::NotFound);
+ ASSERT_OK(fs()->CreateDir(directory_path));
+ AssertFileInfo(fs(), directory_path, FileType::NotFound);
+ ASSERT_OK(fs()->DeleteDir(directory_path));
+ AssertFileInfo(fs(), directory_path, FileType::NotFound);
}
}
void TestCreateDirSuccessContainerAndDirectory() {
auto data = SetUpPreexistingData();
const auto path = data.RandomDirectoryPath(rng_);
- ASSERT_OK(fs_->CreateDir(path, false));
+ ASSERT_OK(fs()->CreateDir(path, false));
if (WithHierarchicalNamespace()) {
- arrow::fs::AssertFileInfo(fs_.get(), path, FileType::Directory);
+ AssertFileInfo(fs(), path, FileType::Directory);
} else {
// There is only virtual directory without hierarchical namespace
// support. So the CreateDir() does nothing.
- arrow::fs::AssertFileInfo(fs_.get(), path, FileType::NotFound);
+ AssertFileInfo(fs(), path, FileType::NotFound);
}
}
void TestCreateDirRecursiveSuccessContainerOnly() {
auto container_name = PreexistingData::RandomContainerName(rng_);
- ASSERT_OK(fs_->CreateDir(container_name, true));
- arrow::fs::AssertFileInfo(fs_.get(), container_name, FileType::Directory);
+ ASSERT_OK(fs()->CreateDir(container_name, true));
+ AssertFileInfo(fs(), container_name, FileType::Directory);
}
void TestCreateDirRecursiveSuccessDirectoryOnly() {
auto data = SetUpPreexistingData();
const auto parent = data.RandomDirectoryPath(rng_);
const auto path = ConcatAbstractPath(parent, "new-sub");
- ASSERT_OK(fs_->CreateDir(path, true));
+ ASSERT_OK(fs()->CreateDir(path, true));
if (WithHierarchicalNamespace()) {
- arrow::fs::AssertFileInfo(fs_.get(), path, FileType::Directory);
- arrow::fs::AssertFileInfo(fs_.get(), parent, FileType::Directory);
+ AssertFileInfo(fs(), path, FileType::Directory);
+ AssertFileInfo(fs(), parent, FileType::Directory);
} else {
// There is only virtual directory without hierarchical namespace
// support. So the CreateDir() does nothing.
- arrow::fs::AssertFileInfo(fs_.get(), path, FileType::NotFound);
- arrow::fs::AssertFileInfo(fs_.get(), parent, FileType::NotFound);
+ AssertFileInfo(fs(), path, FileType::NotFound);
+ AssertFileInfo(fs(), parent, FileType::NotFound);
}
}
@@ -591,31 +602,31 @@ class TestAzureFileSystem : public ::testing::Test {
auto data = SetUpPreexistingData();
const auto parent = data.RandomDirectoryPath(rng_);
const auto path = ConcatAbstractPath(parent, "new-sub");
- ASSERT_OK(fs_->CreateDir(path, true));
+ ASSERT_OK(fs()->CreateDir(path, true));
if (WithHierarchicalNamespace()) {
- arrow::fs::AssertFileInfo(fs_.get(), path, FileType::Directory);
- arrow::fs::AssertFileInfo(fs_.get(), parent, FileType::Directory);
- arrow::fs::AssertFileInfo(fs_.get(), data.container_name,
FileType::Directory);
+ AssertFileInfo(fs(), path, FileType::Directory);
+ AssertFileInfo(fs(), parent, FileType::Directory);
+ AssertFileInfo(fs(), data.container_name, FileType::Directory);
} else {
// There is only virtual directory without hierarchical namespace
// support. So the CreateDir() does nothing.
- arrow::fs::AssertFileInfo(fs_.get(), path, FileType::NotFound);
- arrow::fs::AssertFileInfo(fs_.get(), parent, FileType::NotFound);
- arrow::fs::AssertFileInfo(fs_.get(), data.container_name,
FileType::Directory);
+ AssertFileInfo(fs(), path, FileType::NotFound);
+ AssertFileInfo(fs(), parent, FileType::NotFound);
+ AssertFileInfo(fs(), data.container_name, FileType::Directory);
}
}
void TestDeleteDirContentsSuccessNonexistent() {
auto data = SetUpPreexistingData();
const auto directory_path = data.RandomDirectoryPath(rng_);
- ASSERT_OK(fs_->DeleteDirContents(directory_path, true));
- arrow::fs::AssertFileInfo(fs_.get(), directory_path, FileType::NotFound);
+ ASSERT_OK(fs()->DeleteDirContents(directory_path, true));
+ AssertFileInfo(fs(), directory_path, FileType::NotFound);
}
void TestDeleteDirContentsFailureNonexistent() {
auto data = SetUpPreexistingData();
const auto directory_path = data.RandomDirectoryPath(rng_);
- ASSERT_RAISES(IOError, fs_->DeleteDirContents(directory_path, false));
+ ASSERT_RAISES(IOError, fs()->DeleteDirContents(directory_path, false));
}
};
@@ -672,12 +683,12 @@ void TestAzureFileSystem::TestGetFileInfoObject() {
.GetProperties()
.Value;
- AssertFileInfo(fs_.get(), data.ObjectPath(), FileType::File,
+ AssertFileInfo(fs(), data.ObjectPath(), FileType::File,
std::chrono::system_clock::time_point{object_properties.LastModified},
static_cast<int64_t>(object_properties.BlobSize));
// URI
- ASSERT_RAISES(Invalid, fs_->GetFileInfo("abfs://" +
std::string{data.kObjectName}));
+ ASSERT_RAISES(Invalid, fs()->GetFileInfo("abfs://" +
std::string{data.kObjectName}));
}
void TestAzureFileSystem::TestGetFileInfoObjectWithNestedStructure() {
@@ -685,37 +696,37 @@ void
TestAzureFileSystem::TestGetFileInfoObjectWithNestedStructure() {
// Adds detailed tests to handle cases of different edge cases
// with directory naming conventions (e.g. with and without slashes).
const std::string kObjectName =
"test-object-dir/some_other_dir/another_dir/foo";
- ASSERT_OK_AND_ASSIGN(auto output,
fs_->OpenOutputStream(data.ContainerPath(kObjectName),
- /*metadata=*/{}));
+ ASSERT_OK_AND_ASSIGN(auto output,
+ fs()->OpenOutputStream(data.ContainerPath(kObjectName),
+ /*metadata=*/{}));
const std::string_view lorem_ipsum(PreexistingData::kLoremIpsum);
ASSERT_OK(output->Write(lorem_ipsum));
ASSERT_OK(output->Close());
// 0 is immediately after "/" lexicographically, ensure that this doesn't
// cause unexpected issues.
- ASSERT_OK_AND_ASSIGN(
- output,
fs_->OpenOutputStream(data.ContainerPath("test-object-dir/some_other_dir0"),
- /*metadata=*/{}));
+ ASSERT_OK_AND_ASSIGN(output, fs()->OpenOutputStream(
+
data.ContainerPath("test-object-dir/some_other_dir0"),
+ /*metadata=*/{}));
ASSERT_OK(output->Write(lorem_ipsum));
ASSERT_OK(output->Close());
ASSERT_OK_AND_ASSIGN(output,
- fs_->OpenOutputStream(data.ContainerPath(kObjectName +
"0"),
- /*metadata=*/{}));
+ fs()->OpenOutputStream(data.ContainerPath(kObjectName +
"0"),
+ /*metadata=*/{}));
ASSERT_OK(output->Write(lorem_ipsum));
ASSERT_OK(output->Close());
- AssertFileInfo(fs_.get(), data.ContainerPath(kObjectName), FileType::File);
- AssertFileInfo(fs_.get(), data.ContainerPath(kObjectName) + "/",
FileType::NotFound);
- AssertFileInfo(fs_.get(), data.ContainerPath("test-object-dir"),
FileType::Directory);
- AssertFileInfo(fs_.get(), data.ContainerPath("test-object-dir") + "/",
- FileType::Directory);
- AssertFileInfo(fs_.get(),
data.ContainerPath("test-object-dir/some_other_dir"),
+ AssertFileInfo(fs(), data.ContainerPath(kObjectName), FileType::File);
+ AssertFileInfo(fs(), data.ContainerPath(kObjectName) + "/",
FileType::NotFound);
+ AssertFileInfo(fs(), data.ContainerPath("test-object-dir"),
FileType::Directory);
+ AssertFileInfo(fs(), data.ContainerPath("test-object-dir") + "/",
FileType::Directory);
+ AssertFileInfo(fs(), data.ContainerPath("test-object-dir/some_other_dir"),
FileType::Directory);
- AssertFileInfo(fs_.get(),
data.ContainerPath("test-object-dir/some_other_dir") + "/",
+ AssertFileInfo(fs(), data.ContainerPath("test-object-dir/some_other_dir") +
"/",
FileType::Directory);
- AssertFileInfo(fs_.get(), data.ContainerPath("test-object-di"),
FileType::NotFound);
- AssertFileInfo(fs_.get(),
data.ContainerPath("test-object-dir/some_other_di"),
+ AssertFileInfo(fs(), data.ContainerPath("test-object-di"),
FileType::NotFound);
+ AssertFileInfo(fs(), data.ContainerPath("test-object-dir/some_other_di"),
FileType::NotFound);
if (WithHierarchicalNamespace()) {
@@ -723,17 +734,45 @@ void
TestAzureFileSystem::TestGetFileInfoObjectWithNestedStructure() {
.GetDirectoryClient("test-empty-object-dir")
.Create();
- AssertFileInfo(fs_.get(), data.ContainerPath("test-empty-object-dir"),
+ AssertFileInfo(fs(), data.ContainerPath("test-empty-object-dir"),
FileType::Directory);
}
}
-template <class AzureEnvClass>
+template <class AzureEnv, bool HNSSupportShouldBeKnown = false>
+struct TestingScenario {
+ using AzureEnvClass = AzureEnv;
+ static constexpr bool kHNSSupportShouldBeKnown = HNSSupportShouldBeKnown;
+};
+
+template <class TestingScenario>
class AzureFileSystemTestImpl : public TestAzureFileSystem {
public:
+ using AzureEnvClass = typename TestingScenario::AzureEnvClass;
+
using TestAzureFileSystem::TestAzureFileSystem;
Result<BaseAzureEnv*> GetAzureEnv() const final { return
AzureEnvClass::GetInstance(); }
+
+ /// \brief HNSSupport value that should be assumed as the cached
+ /// HNSSupport on every fs()->Operation(...) call in tests.
+ ///
+ /// If TestingScenario::kHNSSupportShouldBeKnown is true, this value
+ /// will be HNSSupport::kEnabled or HNSSupport::kDisabled, depending
+ /// on the environment. Otherwise, this value will be HNSSupport::kUnknown.
+ ///
+ /// This ensures all the branches in the AzureFileSystem code operations are
tested.
+ /// For instance, many operations executed on a missing container, wouldn't
+ /// get a HNSSupport::kContainerNotFound error if the cached HNSSupport was
+ /// already known due to a previous operation that cached the HNSSupport
value.
+ HNSSupport CachedHNSSupport(const BaseAzureEnv& env) const final {
+ if constexpr (TestingScenario::kHNSSupportShouldBeKnown) {
+ return env.WithHierarchicalNamespace() ? HNSSupport::kEnabled
+ : HNSSupport::kDisabled;
+ } else {
+ return HNSSupport::kUnknown;
+ }
+ }
};
// How to enable the non-Azurite tests:
@@ -762,54 +801,71 @@ class AzureFileSystemTestImpl : public
TestAzureFileSystem {
// [1]: https://azure.microsoft.com/en-gb/free/
// [2]:
//
https://learn.microsoft.com/en-us/azure/storage/blobs/create-data-lake-storage-account
-using TestAzureFlatNSFileSystem = AzureFileSystemTestImpl<AzureFlatNSEnv>;
-using TestAzureHierarchicalNSFileSystem =
AzureFileSystemTestImpl<AzureHierarchicalNSEnv>;
-using TestAzuriteFileSystem = AzureFileSystemTestImpl<AzuriteEnv>;
+using TestAzureFlatNSFileSystem =
+ AzureFileSystemTestImpl<TestingScenario<AzureFlatNSEnv>>;
+using TestAzureHierarchicalNSFileSystem =
+ AzureFileSystemTestImpl<TestingScenario<AzureHierarchicalNSEnv>>;
+using TestAzuriteFileSystem =
AzureFileSystemTestImpl<TestingScenario<AzuriteEnv>>;
-// Tests using all the 3 environments (Azurite, Azure w/o HNS (flat), Azure w/
HNS)
-
-template <class AzureEnvClass>
-using AzureFileSystemTestOnAllEnvs = AzureFileSystemTestImpl<AzureEnvClass>;
+// Tests using all the 3 environments (Azurite, Azure w/o HNS (flat), Azure w/
HNS).
+template <class TestingScenario>
+using TestAzureFileSystemOnAllEnvs = AzureFileSystemTestImpl<TestingScenario>;
using AllEnvironments =
- ::testing::Types<AzuriteEnv, AzureFlatNSEnv, AzureHierarchicalNSEnv>;
+ ::testing::Types<TestingScenario<AzuriteEnv>,
TestingScenario<AzureFlatNSEnv>,
+ TestingScenario<AzureHierarchicalNSEnv>>;
-TYPED_TEST_SUITE(AzureFileSystemTestOnAllEnvs, AllEnvironments);
+TYPED_TEST_SUITE(TestAzureFileSystemOnAllEnvs, AllEnvironments);
-TYPED_TEST(AzureFileSystemTestOnAllEnvs, DetectHierarchicalNamespace) {
+TYPED_TEST(TestAzureFileSystemOnAllEnvs, DetectHierarchicalNamespace) {
this->TestDetectHierarchicalNamespace(true);
this->TestDetectHierarchicalNamespace(false);
}
-TYPED_TEST(AzureFileSystemTestOnAllEnvs,
DetectHierarchicalNamespaceOnMissingContainer) {
+TYPED_TEST(TestAzureFileSystemOnAllEnvs,
DetectHierarchicalNamespaceOnMissingContainer) {
this->TestDetectHierarchicalNamespaceOnMissingContainer();
}
-TYPED_TEST(AzureFileSystemTestOnAllEnvs, GetFileInfoObject) {
+// Tests using all the 3 environments (Azurite, Azure w/o HNS (flat), Azure w/
HNS)
+// combined with the two scenarios for AzureFileSystem::cached_hns_support_ --
unknown and
+// known according to the environment.
+template <class TestingScenario>
+using TestAzureFileSystemOnAllScenarios =
AzureFileSystemTestImpl<TestingScenario>;
+
+using AllScenarios = ::testing::Types<
+ TestingScenario<AzuriteEnv, true>, TestingScenario<AzuriteEnv, false>,
+ TestingScenario<AzureFlatNSEnv, true>, TestingScenario<AzureFlatNSEnv,
false>,
+ TestingScenario<AzureHierarchicalNSEnv, true>,
+ TestingScenario<AzureHierarchicalNSEnv, false>>;
+
+TYPED_TEST_SUITE(TestAzureFileSystemOnAllScenarios, AllScenarios);
+
+TYPED_TEST(TestAzureFileSystemOnAllScenarios, GetFileInfoObject) {
this->TestGetFileInfoObject();
}
-TYPED_TEST(AzureFileSystemTestOnAllEnvs, DeleteDirSuccessEmpty) {
+TYPED_TEST(TestAzureFileSystemOnAllScenarios, DeleteDirSuccessEmpty) {
this->TestDeleteDirSuccessEmpty();
}
-TYPED_TEST(AzureFileSystemTestOnAllEnvs, GetFileInfoObjectWithNestedStructure)
{
+TYPED_TEST(TestAzureFileSystemOnAllScenarios,
GetFileInfoObjectWithNestedStructure) {
this->TestGetFileInfoObjectWithNestedStructure();
}
-TYPED_TEST(AzureFileSystemTestOnAllEnvs,
CreateDirSuccessContainerAndDirectory) {
+TYPED_TEST(TestAzureFileSystemOnAllScenarios,
CreateDirSuccessContainerAndDirectory) {
this->TestCreateDirSuccessContainerAndDirectory();
}
-TYPED_TEST(AzureFileSystemTestOnAllEnvs,
CreateDirRecursiveSuccessContainerOnly) {
+TYPED_TEST(TestAzureFileSystemOnAllScenarios,
CreateDirRecursiveSuccessContainerOnly) {
this->TestCreateDirRecursiveSuccessContainerOnly();
}
-TYPED_TEST(AzureFileSystemTestOnAllEnvs,
CreateDirRecursiveSuccessDirectoryOnly) {
+TYPED_TEST(TestAzureFileSystemOnAllScenarios,
CreateDirRecursiveSuccessDirectoryOnly) {
this->TestCreateDirRecursiveSuccessDirectoryOnly();
}
-TYPED_TEST(AzureFileSystemTestOnAllEnvs,
CreateDirRecursiveSuccessContainerAndDirectory) {
+TYPED_TEST(TestAzureFileSystemOnAllScenarios,
+ CreateDirRecursiveSuccessContainerAndDirectory) {
this->TestCreateDirRecursiveSuccessContainerAndDirectory();
}
@@ -818,41 +874,41 @@ TYPED_TEST(AzureFileSystemTestOnAllEnvs,
CreateDirRecursiveSuccessContainerAndDi
TEST_F(TestAzureHierarchicalNSFileSystem, DeleteDirFailureNonexistent) {
auto data = SetUpPreexistingData();
const auto path = data.RandomDirectoryPath(rng_);
- ASSERT_RAISES(IOError, fs_->DeleteDir(path));
+ ASSERT_RAISES(IOError, fs()->DeleteDir(path));
}
TEST_F(TestAzureHierarchicalNSFileSystem, DeleteDirSuccessHaveBlob) {
auto data = SetUpPreexistingData();
const auto directory_path = data.RandomDirectoryPath(rng_);
const auto blob_path = ConcatAbstractPath(directory_path, "hello.txt");
- ASSERT_OK_AND_ASSIGN(auto output, fs_->OpenOutputStream(blob_path));
+ ASSERT_OK_AND_ASSIGN(auto output, fs()->OpenOutputStream(blob_path));
ASSERT_OK(output->Write(std::string_view("hello")));
ASSERT_OK(output->Close());
- arrow::fs::AssertFileInfo(fs_.get(), blob_path, FileType::File);
- ASSERT_OK(fs_->DeleteDir(directory_path));
- arrow::fs::AssertFileInfo(fs_.get(), blob_path, FileType::NotFound);
+ AssertFileInfo(fs(), blob_path, FileType::File);
+ ASSERT_OK(fs()->DeleteDir(directory_path));
+ AssertFileInfo(fs(), blob_path, FileType::NotFound);
}
TEST_F(TestAzureHierarchicalNSFileSystem, DeleteDirSuccessHaveDirectory) {
auto data = SetUpPreexistingData();
const auto parent = data.RandomDirectoryPath(rng_);
const auto path = ConcatAbstractPath(parent, "new-sub");
- ASSERT_OK(fs_->CreateDir(path, true));
- arrow::fs::AssertFileInfo(fs_.get(), path, FileType::Directory);
- arrow::fs::AssertFileInfo(fs_.get(), parent, FileType::Directory);
- ASSERT_OK(fs_->DeleteDir(parent));
- arrow::fs::AssertFileInfo(fs_.get(), path, FileType::NotFound);
- arrow::fs::AssertFileInfo(fs_.get(), parent, FileType::NotFound);
+ ASSERT_OK(fs()->CreateDir(path, true));
+ AssertFileInfo(fs(), path, FileType::Directory);
+ AssertFileInfo(fs(), parent, FileType::Directory);
+ ASSERT_OK(fs()->DeleteDir(parent));
+ AssertFileInfo(fs(), path, FileType::NotFound);
+ AssertFileInfo(fs(), parent, FileType::NotFound);
}
TEST_F(TestAzureHierarchicalNSFileSystem, DeleteDirContentsSuccessExist) {
auto preexisting_data = SetUpPreexistingData();
HierarchicalPaths paths;
CreateHierarchicalData(&paths);
- ASSERT_OK(fs_->DeleteDirContents(paths.directory));
- arrow::fs::AssertFileInfo(fs_.get(), paths.directory, FileType::Directory);
+ ASSERT_OK(fs()->DeleteDirContents(paths.directory));
+ AssertFileInfo(fs(), paths.directory, FileType::Directory);
for (const auto& sub_path : paths.sub_paths) {
- arrow::fs::AssertFileInfo(fs_.get(), sub_path, FileType::NotFound);
+ AssertFileInfo(fs(), sub_path, FileType::NotFound);
}
}
@@ -867,20 +923,20 @@ TEST_F(TestAzureHierarchicalNSFileSystem,
DeleteDirContentsFailureNonexistent) {
// Tests using Azurite (the local Azure emulator)
TEST_F(TestAzuriteFileSystem, GetFileInfoAccount) {
- AssertFileInfo(fs_.get(), "", FileType::Directory);
+ AssertFileInfo(fs(), "", FileType::Directory);
// URI
- ASSERT_RAISES(Invalid, fs_->GetFileInfo("abfs://"));
+ ASSERT_RAISES(Invalid, fs()->GetFileInfo("abfs://"));
}
TEST_F(TestAzuriteFileSystem, GetFileInfoContainer) {
auto data = SetUpPreexistingData();
- AssertFileInfo(fs_.get(), data.container_name, FileType::Directory);
+ AssertFileInfo(fs(), data.container_name, FileType::Directory);
- AssertFileInfo(fs_.get(), "nonexistent-container", FileType::NotFound);
+ AssertFileInfo(fs(), "nonexistent-container", FileType::NotFound);
// URI
- ASSERT_RAISES(Invalid, fs_->GetFileInfo("abfs://" + data.container_name));
+ ASSERT_RAISES(Invalid, fs()->GetFileInfo("abfs://" + data.container_name));
}
TEST_F(TestAzuriteFileSystem, GetFileInfoSelector) {
@@ -891,7 +947,7 @@ TEST_F(TestAzuriteFileSystem, GetFileInfoSelector) {
// Root dir
select.base_dir = "";
- ASSERT_OK_AND_ASSIGN(infos, fs_->GetFileInfo(select));
+ ASSERT_OK_AND_ASSIGN(infos, fs()->GetFileInfo(select));
ASSERT_EQ(infos.size(), 2);
ASSERT_EQ(infos, SortedInfos(infos));
AssertFileInfo(infos[0], "container", FileType::Directory);
@@ -899,18 +955,18 @@ TEST_F(TestAzuriteFileSystem, GetFileInfoSelector) {
// Empty container
select.base_dir = "empty-container";
- ASSERT_OK_AND_ASSIGN(infos, fs_->GetFileInfo(select));
+ ASSERT_OK_AND_ASSIGN(infos, fs()->GetFileInfo(select));
ASSERT_EQ(infos.size(), 0);
// Nonexistent container
select.base_dir = "nonexistent-container";
- ASSERT_RAISES(IOError, fs_->GetFileInfo(select));
+ ASSERT_RAISES(IOError, fs()->GetFileInfo(select));
select.allow_not_found = true;
- ASSERT_OK_AND_ASSIGN(infos, fs_->GetFileInfo(select));
+ ASSERT_OK_AND_ASSIGN(infos, fs()->GetFileInfo(select));
ASSERT_EQ(infos.size(), 0);
select.allow_not_found = false;
// Non-empty container
select.base_dir = "container";
- ASSERT_OK_AND_ASSIGN(infos, fs_->GetFileInfo(select));
+ ASSERT_OK_AND_ASSIGN(infos, fs()->GetFileInfo(select));
ASSERT_EQ(infos, SortedInfos(infos));
ASSERT_EQ(infos.size(), 4);
AssertFileInfo(infos[0], "container/emptydir", FileType::Directory);
@@ -920,33 +976,33 @@ TEST_F(TestAzuriteFileSystem, GetFileInfoSelector) {
// Empty "directory"
select.base_dir = "container/emptydir";
- ASSERT_OK_AND_ASSIGN(infos, fs_->GetFileInfo(select));
+ ASSERT_OK_AND_ASSIGN(infos, fs()->GetFileInfo(select));
ASSERT_EQ(infos.size(), 0);
// Non-empty "directories"
select.base_dir = "container/somedir";
- ASSERT_OK_AND_ASSIGN(infos, fs_->GetFileInfo(select));
+ ASSERT_OK_AND_ASSIGN(infos, fs()->GetFileInfo(select));
ASSERT_EQ(infos.size(), 1);
AssertFileInfo(infos[0], "container/somedir/subdir", FileType::Directory);
select.base_dir = "container/somedir/subdir";
- ASSERT_OK_AND_ASSIGN(infos, fs_->GetFileInfo(select));
+ ASSERT_OK_AND_ASSIGN(infos, fs()->GetFileInfo(select));
ASSERT_EQ(infos.size(), 1);
AssertFileInfo(infos[0], "container/somedir/subdir/subfile", FileType::File,
8);
// Nonexistent
select.base_dir = "container/nonexistent";
- ASSERT_RAISES(IOError, fs_->GetFileInfo(select));
+ ASSERT_RAISES(IOError, fs()->GetFileInfo(select));
select.allow_not_found = true;
- ASSERT_OK_AND_ASSIGN(infos, fs_->GetFileInfo(select));
+ ASSERT_OK_AND_ASSIGN(infos, fs()->GetFileInfo(select));
ASSERT_EQ(infos.size(), 0);
select.allow_not_found = false;
// Trailing slashes
select.base_dir = "empty-container/";
- ASSERT_OK_AND_ASSIGN(infos, fs_->GetFileInfo(select));
+ ASSERT_OK_AND_ASSIGN(infos, fs()->GetFileInfo(select));
ASSERT_EQ(infos.size(), 0);
select.base_dir = "nonexistent-container/";
- ASSERT_RAISES(IOError, fs_->GetFileInfo(select));
+ ASSERT_RAISES(IOError, fs()->GetFileInfo(select));
select.base_dir = "container/";
- ASSERT_OK_AND_ASSIGN(infos, fs_->GetFileInfo(select));
+ ASSERT_OK_AND_ASSIGN(infos, fs()->GetFileInfo(select));
ASSERT_EQ(infos, SortedInfos(infos));
ASSERT_EQ(infos.size(), 4);
}
@@ -960,19 +1016,19 @@ TEST_F(TestAzuriteFileSystem,
GetFileInfoSelectorRecursive) {
std::vector<FileInfo> infos;
// Root dir
select.base_dir = "";
- ASSERT_OK_AND_ASSIGN(infos, fs_->GetFileInfo(select));
+ ASSERT_OK_AND_ASSIGN(infos, fs()->GetFileInfo(select));
ASSERT_EQ(infos.size(), 12);
ASSERT_EQ(infos, SortedInfos(infos));
AssertInfoAllContainersRecursive(infos);
// Empty container
select.base_dir = "empty-container";
- ASSERT_OK_AND_ASSIGN(infos, fs_->GetFileInfo(select));
+ ASSERT_OK_AND_ASSIGN(infos, fs()->GetFileInfo(select));
ASSERT_EQ(infos.size(), 0);
// Non-empty container
select.base_dir = "container";
- ASSERT_OK_AND_ASSIGN(infos, fs_->GetFileInfo(select));
+ ASSERT_OK_AND_ASSIGN(infos, fs()->GetFileInfo(select));
ASSERT_EQ(infos, SortedInfos(infos));
ASSERT_EQ(infos.size(), 10);
AssertFileInfo(infos[0], "container/emptydir", FileType::Directory);
@@ -988,19 +1044,19 @@ TEST_F(TestAzuriteFileSystem,
GetFileInfoSelectorRecursive) {
// Empty "directory"
select.base_dir = "container/emptydir";
- ASSERT_OK_AND_ASSIGN(infos, fs_->GetFileInfo(select));
+ ASSERT_OK_AND_ASSIGN(infos, fs()->GetFileInfo(select));
ASSERT_EQ(infos.size(), 0);
// Non-empty "directories"
select.base_dir = "container/somedir";
- ASSERT_OK_AND_ASSIGN(infos, fs_->GetFileInfo(select));
+ ASSERT_OK_AND_ASSIGN(infos, fs()->GetFileInfo(select));
ASSERT_EQ(infos, SortedInfos(infos));
ASSERT_EQ(infos.size(), 2);
AssertFileInfo(infos[0], "container/somedir/subdir", FileType::Directory);
AssertFileInfo(infos[1], "container/somedir/subdir/subfile", FileType::File,
8);
select.base_dir = "container/otherdir";
- ASSERT_OK_AND_ASSIGN(infos, fs_->GetFileInfo(select));
+ ASSERT_OK_AND_ASSIGN(infos, fs()->GetFileInfo(select));
ASSERT_EQ(infos, SortedInfos(infos));
ASSERT_EQ(infos.size(), 4);
AssertFileInfo(infos[0], "container/otherdir/1", FileType::Directory);
@@ -1023,13 +1079,13 @@ TEST_F(TestAzuriteFileSystem,
GetFileInfoSelectorExplicitImplicitDirDedup) {
FileSelector select; // non-recursive
select.base_dir = "container";
- ASSERT_OK_AND_ASSIGN(infos, fs_->GetFileInfo(select));
+ ASSERT_OK_AND_ASSIGN(infos, fs()->GetFileInfo(select));
ASSERT_EQ(infos.size(), 1);
ASSERT_EQ(infos, SortedInfos(infos));
AssertFileInfo(infos[0], "container/mydir", FileType::Directory);
select.base_dir = "container/mydir";
- ASSERT_OK_AND_ASSIGN(infos, fs_->GetFileInfo(select));
+ ASSERT_OK_AND_ASSIGN(infos, fs()->GetFileInfo(select));
ASSERT_EQ(infos.size(), 4);
ASSERT_EQ(infos, SortedInfos(infos));
AssertFileInfo(infos[0], "container/mydir/emptydir1", FileType::Directory);
@@ -1038,55 +1094,55 @@ TEST_F(TestAzuriteFileSystem,
GetFileInfoSelectorExplicitImplicitDirDedup) {
AssertFileInfo(infos[3], "container/mydir/nonemptydir2",
FileType::Directory);
select.base_dir = "container/mydir/emptydir1";
- ASSERT_OK_AND_ASSIGN(infos, fs_->GetFileInfo(select));
+ ASSERT_OK_AND_ASSIGN(infos, fs()->GetFileInfo(select));
ASSERT_EQ(infos.size(), 0);
select.base_dir = "container/mydir/emptydir2";
- ASSERT_OK_AND_ASSIGN(infos, fs_->GetFileInfo(select));
+ ASSERT_OK_AND_ASSIGN(infos, fs()->GetFileInfo(select));
ASSERT_EQ(infos.size(), 0);
select.base_dir = "container/mydir/nonemptydir1";
- ASSERT_OK_AND_ASSIGN(infos, fs_->GetFileInfo(select));
+ ASSERT_OK_AND_ASSIGN(infos, fs()->GetFileInfo(select));
ASSERT_EQ(infos.size(), 1);
AssertFileInfo(infos[0], "container/mydir/nonemptydir1/somefile",
FileType::File);
select.base_dir = "container/mydir/nonemptydir2";
- ASSERT_OK_AND_ASSIGN(infos, fs_->GetFileInfo(select));
+ ASSERT_OK_AND_ASSIGN(infos, fs()->GetFileInfo(select));
ASSERT_EQ(infos.size(), 1);
AssertFileInfo(infos[0], "container/mydir/nonemptydir2/somefile",
FileType::File);
}
TEST_F(TestAzuriteFileSystem, CreateDirFailureNoContainer) {
- ASSERT_RAISES(Invalid, fs_->CreateDir("", false));
+ ASSERT_RAISES(Invalid, fs()->CreateDir("", false));
}
TEST_F(TestAzuriteFileSystem, CreateDirSuccessContainerOnly) {
auto container_name = PreexistingData::RandomContainerName(rng_);
- ASSERT_OK(fs_->CreateDir(container_name, false));
- arrow::fs::AssertFileInfo(fs_.get(), container_name, FileType::Directory);
+ ASSERT_OK(fs()->CreateDir(container_name, false));
+ AssertFileInfo(fs(), container_name, FileType::Directory);
}
TEST_F(TestAzuriteFileSystem, CreateDirFailureDirectoryWithMissingContainer) {
const auto path = std::string("not-a-container/new-directory");
- ASSERT_RAISES(IOError, fs_->CreateDir(path, false));
+ ASSERT_RAISES(IOError, fs()->CreateDir(path, false));
}
TEST_F(TestAzuriteFileSystem, CreateDirRecursiveFailureNoContainer) {
- ASSERT_RAISES(Invalid, fs_->CreateDir("", true));
+ ASSERT_RAISES(Invalid, fs()->CreateDir("", true));
}
TEST_F(TestAzuriteFileSystem, CreateDirUri) {
ASSERT_RAISES(
Invalid,
- fs_->CreateDir("abfs://" + PreexistingData::RandomContainerName(rng_),
true));
+ fs()->CreateDir("abfs://" + PreexistingData::RandomContainerName(rng_),
true));
}
TEST_F(TestAzuriteFileSystem, DeleteDirSuccessContainer) {
const auto container_name = PreexistingData::RandomContainerName(rng_);
- ASSERT_OK(fs_->CreateDir(container_name));
- arrow::fs::AssertFileInfo(fs_.get(), container_name, FileType::Directory);
- ASSERT_OK(fs_->DeleteDir(container_name));
- arrow::fs::AssertFileInfo(fs_.get(), container_name, FileType::NotFound);
+ ASSERT_OK(fs()->CreateDir(container_name));
+ AssertFileInfo(fs(), container_name, FileType::Directory);
+ ASSERT_OK(fs()->DeleteDir(container_name));
+ AssertFileInfo(fs(), container_name, FileType::NotFound);
}
TEST_F(TestAzuriteFileSystem, DeleteDirSuccessNonexistent) {
@@ -1094,8 +1150,8 @@ TEST_F(TestAzuriteFileSystem,
DeleteDirSuccessNonexistent) {
const auto directory_path = data.RandomDirectoryPath(rng_);
// There is only virtual directory without hierarchical namespace
// support. So the DeleteDir() for nonexistent directory does nothing.
- ASSERT_OK(fs_->DeleteDir(directory_path));
- arrow::fs::AssertFileInfo(fs_.get(), directory_path, FileType::NotFound);
+ ASSERT_OK(fs()->DeleteDir(directory_path));
+ AssertFileInfo(fs(), directory_path, FileType::NotFound);
}
TEST_F(TestAzuriteFileSystem, DeleteDirSuccessHaveBlobs) {
@@ -1110,21 +1166,21 @@ TEST_F(TestAzuriteFileSystem,
DeleteDirSuccessHaveBlobs) {
int64_t n_blobs = 257;
for (int64_t i = 0; i < n_blobs; ++i) {
const auto blob_path = ConcatAbstractPath(directory_path,
std::to_string(i) + ".txt");
- ASSERT_OK_AND_ASSIGN(auto output, fs_->OpenOutputStream(blob_path));
+ ASSERT_OK_AND_ASSIGN(auto output, fs()->OpenOutputStream(blob_path));
ASSERT_OK(output->Write(std::string_view(std::to_string(i))));
ASSERT_OK(output->Close());
- arrow::fs::AssertFileInfo(fs_.get(), blob_path, FileType::File);
+ AssertFileInfo(fs(), blob_path, FileType::File);
}
- ASSERT_OK(fs_->DeleteDir(directory_path));
+ ASSERT_OK(fs()->DeleteDir(directory_path));
for (int64_t i = 0; i < n_blobs; ++i) {
const auto blob_path = ConcatAbstractPath(directory_path,
std::to_string(i) + ".txt");
- arrow::fs::AssertFileInfo(fs_.get(), blob_path, FileType::NotFound);
+ AssertFileInfo(fs(), blob_path, FileType::NotFound);
}
}
TEST_F(TestAzuriteFileSystem, DeleteDirUri) {
auto data = SetUpPreexistingData();
- ASSERT_RAISES(Invalid, fs_->DeleteDir("abfs://" + data.container_name +
"/"));
+ ASSERT_RAISES(Invalid, fs()->DeleteDir("abfs://" + data.container_name +
"/"));
}
TEST_F(TestAzuriteFileSystem, DeleteDirContentsSuccessContainer) {
@@ -1135,11 +1191,11 @@ TEST_F(TestAzuriteFileSystem,
DeleteDirContentsSuccessContainer) {
auto data = SetUpPreexistingData();
HierarchicalPaths paths;
CreateHierarchicalData(&paths);
- ASSERT_OK(fs_->DeleteDirContents(paths.container));
- arrow::fs::AssertFileInfo(fs_.get(), paths.container, FileType::Directory);
- arrow::fs::AssertFileInfo(fs_.get(), paths.directory, FileType::NotFound);
+ ASSERT_OK(fs()->DeleteDirContents(paths.container));
+ AssertFileInfo(fs(), paths.container, FileType::Directory);
+ AssertFileInfo(fs(), paths.directory, FileType::NotFound);
for (const auto& sub_path : paths.sub_paths) {
- arrow::fs::AssertFileInfo(fs_.get(), sub_path, FileType::NotFound);
+ AssertFileInfo(fs(), sub_path, FileType::NotFound);
}
}
@@ -1151,11 +1207,11 @@ TEST_F(TestAzuriteFileSystem,
DeleteDirContentsSuccessDirectory) {
auto data = SetUpPreexistingData();
HierarchicalPaths paths;
CreateHierarchicalData(&paths);
- ASSERT_OK(fs_->DeleteDirContents(paths.directory));
+ ASSERT_OK(fs()->DeleteDirContents(paths.directory));
// GH-38772: We may change this to FileType::Directory.
- arrow::fs::AssertFileInfo(fs_.get(), paths.directory, FileType::NotFound);
+ AssertFileInfo(fs(), paths.directory, FileType::NotFound);
for (const auto& sub_path : paths.sub_paths) {
- arrow::fs::AssertFileInfo(fs_.get(), sub_path, FileType::NotFound);
+ AssertFileInfo(fs(), sub_path, FileType::NotFound);
}
}
@@ -1170,52 +1226,52 @@ TEST_F(TestAzuriteFileSystem,
DeleteDirContentsFailureNonexistent) {
TEST_F(TestAzuriteFileSystem, CopyFileSuccessDestinationNonexistent) {
auto data = SetUpPreexistingData();
const auto destination_path = data.ContainerPath("copy-destionation");
- ASSERT_OK(fs_->CopyFile(data.ObjectPath(), destination_path));
- ASSERT_OK_AND_ASSIGN(auto info, fs_->GetFileInfo(destination_path));
- ASSERT_OK_AND_ASSIGN(auto stream, fs_->OpenInputStream(info));
+ ASSERT_OK(fs()->CopyFile(data.ObjectPath(), destination_path));
+ ASSERT_OK_AND_ASSIGN(auto info, fs()->GetFileInfo(destination_path));
+ ASSERT_OK_AND_ASSIGN(auto stream, fs()->OpenInputStream(info));
ASSERT_OK_AND_ASSIGN(auto buffer, stream->Read(1024));
EXPECT_EQ(PreexistingData::kLoremIpsum, buffer->ToString());
}
TEST_F(TestAzuriteFileSystem, CopyFileSuccessDestinationSame) {
auto data = SetUpPreexistingData();
- ASSERT_OK(fs_->CopyFile(data.ObjectPath(), data.ObjectPath()));
- ASSERT_OK_AND_ASSIGN(auto info, fs_->GetFileInfo(data.ObjectPath()));
- ASSERT_OK_AND_ASSIGN(auto stream, fs_->OpenInputStream(info));
+ ASSERT_OK(fs()->CopyFile(data.ObjectPath(), data.ObjectPath()));
+ ASSERT_OK_AND_ASSIGN(auto info, fs()->GetFileInfo(data.ObjectPath()));
+ ASSERT_OK_AND_ASSIGN(auto stream, fs()->OpenInputStream(info));
ASSERT_OK_AND_ASSIGN(auto buffer, stream->Read(1024));
EXPECT_EQ(PreexistingData::kLoremIpsum, buffer->ToString());
}
TEST_F(TestAzuriteFileSystem, CopyFileFailureDestinationTrailingSlash) {
auto data = SetUpPreexistingData();
- ASSERT_RAISES(IOError, fs_->CopyFile(data.ObjectPath(),
-
internal::EnsureTrailingSlash(data.ObjectPath())));
+ ASSERT_RAISES(IOError, fs()->CopyFile(data.ObjectPath(),
internal::EnsureTrailingSlash(
+
data.ObjectPath())));
}
TEST_F(TestAzuriteFileSystem, CopyFileFailureSourceNonexistent) {
auto data = SetUpPreexistingData();
const auto destination_path = data.ContainerPath("copy-destionation");
- ASSERT_RAISES(IOError, fs_->CopyFile(data.NotFoundObjectPath(),
destination_path));
+ ASSERT_RAISES(IOError, fs()->CopyFile(data.NotFoundObjectPath(),
destination_path));
}
TEST_F(TestAzuriteFileSystem, CopyFileFailureDestinationParentNonexistent) {
auto data = SetUpPreexistingData();
const auto destination_path =
ConcatAbstractPath(PreexistingData::RandomContainerName(rng_),
"copy-destionation");
- ASSERT_RAISES(IOError, fs_->CopyFile(data.ObjectPath(), destination_path));
+ ASSERT_RAISES(IOError, fs()->CopyFile(data.ObjectPath(), destination_path));
}
TEST_F(TestAzuriteFileSystem, CopyFileUri) {
auto data = SetUpPreexistingData();
const auto destination_path = data.ContainerPath("copy-destionation");
- ASSERT_RAISES(Invalid, fs_->CopyFile("abfs://" + data.ObjectPath(),
destination_path));
- ASSERT_RAISES(Invalid, fs_->CopyFile(data.ObjectPath(), "abfs://" +
destination_path));
+ ASSERT_RAISES(Invalid, fs()->CopyFile("abfs://" + data.ObjectPath(),
destination_path));
+ ASSERT_RAISES(Invalid, fs()->CopyFile(data.ObjectPath(), "abfs://" +
destination_path));
}
TEST_F(TestAzuriteFileSystem, OpenInputStreamString) {
auto data = SetUpPreexistingData();
std::shared_ptr<io::InputStream> stream;
- ASSERT_OK_AND_ASSIGN(stream, fs_->OpenInputStream(data.ObjectPath()));
+ ASSERT_OK_AND_ASSIGN(stream, fs()->OpenInputStream(data.ObjectPath()));
ASSERT_OK_AND_ASSIGN(auto buffer, stream->Read(1024));
EXPECT_EQ(buffer->ToString(), PreexistingData::kLoremIpsum);
@@ -1224,7 +1280,7 @@ TEST_F(TestAzuriteFileSystem, OpenInputStreamString) {
TEST_F(TestAzuriteFileSystem, OpenInputStreamStringBuffers) {
auto data = SetUpPreexistingData();
std::shared_ptr<io::InputStream> stream;
- ASSERT_OK_AND_ASSIGN(stream, fs_->OpenInputStream(data.ObjectPath()));
+ ASSERT_OK_AND_ASSIGN(stream, fs()->OpenInputStream(data.ObjectPath()));
std::string contents;
std::shared_ptr<Buffer> buffer;
@@ -1238,10 +1294,10 @@ TEST_F(TestAzuriteFileSystem,
OpenInputStreamStringBuffers) {
TEST_F(TestAzuriteFileSystem, OpenInputStreamInfo) {
auto data = SetUpPreexistingData();
- ASSERT_OK_AND_ASSIGN(auto info, fs_->GetFileInfo(data.ObjectPath()));
+ ASSERT_OK_AND_ASSIGN(auto info, fs()->GetFileInfo(data.ObjectPath()));
std::shared_ptr<io::InputStream> stream;
- ASSERT_OK_AND_ASSIGN(stream, fs_->OpenInputStream(info));
+ ASSERT_OK_AND_ASSIGN(stream, fs()->OpenInputStream(info));
ASSERT_OK_AND_ASSIGN(auto buffer, stream->Read(1024));
EXPECT_EQ(buffer->ToString(), PreexistingData::kLoremIpsum);
@@ -1255,7 +1311,7 @@ TEST_F(TestAzuriteFileSystem, OpenInputStreamEmpty) {
.GetBlockBlobClient(path_to_file)
.UploadFrom(nullptr, 0);
- ASSERT_OK_AND_ASSIGN(auto stream, fs_->OpenInputStream(path));
+ ASSERT_OK_AND_ASSIGN(auto stream, fs()->OpenInputStream(path));
std::array<char, 1024> buffer{};
std::int64_t size;
ASSERT_OK_AND_ASSIGN(size, stream->Read(buffer.size(), buffer.data()));
@@ -1264,26 +1320,26 @@ TEST_F(TestAzuriteFileSystem, OpenInputStreamEmpty) {
TEST_F(TestAzuriteFileSystem, OpenInputStreamNotFound) {
auto data = SetUpPreexistingData();
- ASSERT_RAISES(IOError, fs_->OpenInputStream(data.NotFoundObjectPath()));
+ ASSERT_RAISES(IOError, fs()->OpenInputStream(data.NotFoundObjectPath()));
}
TEST_F(TestAzuriteFileSystem, OpenInputStreamInfoInvalid) {
auto data = SetUpPreexistingData();
- ASSERT_OK_AND_ASSIGN(auto info, fs_->GetFileInfo(data.container_name + "/"));
- ASSERT_RAISES(IOError, fs_->OpenInputStream(info));
+ ASSERT_OK_AND_ASSIGN(auto info, fs()->GetFileInfo(data.container_name +
"/"));
+ ASSERT_RAISES(IOError, fs()->OpenInputStream(info));
- ASSERT_OK_AND_ASSIGN(auto info2,
fs_->GetFileInfo(data.NotFoundObjectPath()));
- ASSERT_RAISES(IOError, fs_->OpenInputStream(info2));
+ ASSERT_OK_AND_ASSIGN(auto info2,
fs()->GetFileInfo(data.NotFoundObjectPath()));
+ ASSERT_RAISES(IOError, fs()->OpenInputStream(info2));
}
TEST_F(TestAzuriteFileSystem, OpenInputStreamUri) {
auto data = SetUpPreexistingData();
- ASSERT_RAISES(Invalid, fs_->OpenInputStream("abfs://" + data.ObjectPath()));
+ ASSERT_RAISES(Invalid, fs()->OpenInputStream("abfs://" + data.ObjectPath()));
}
TEST_F(TestAzuriteFileSystem, OpenInputStreamTrailingSlash) {
auto data = SetUpPreexistingData();
- ASSERT_RAISES(IOError, fs_->OpenInputStream(data.ObjectPath() + '/'));
+ ASSERT_RAISES(IOError, fs()->OpenInputStream(data.ObjectPath() + '/'));
}
namespace {
@@ -1324,7 +1380,7 @@ std::shared_ptr<const KeyValueMetadata>
NormalizerKeyValueMetadata(
TEST_F(TestAzuriteFileSystem, OpenInputStreamReadMetadata) {
auto data = SetUpPreexistingData();
std::shared_ptr<io::InputStream> stream;
- ASSERT_OK_AND_ASSIGN(stream, fs_->OpenInputStream(data.ObjectPath()));
+ ASSERT_OK_AND_ASSIGN(stream, fs()->OpenInputStream(data.ObjectPath()));
std::shared_ptr<const KeyValueMetadata> actual;
ASSERT_OK_AND_ASSIGN(actual, stream->ReadMetadata());
@@ -1354,7 +1410,7 @@ TEST_F(TestAzuriteFileSystem,
OpenInputStreamReadMetadata) {
TEST_F(TestAzuriteFileSystem, OpenInputStreamClosed) {
auto data = SetUpPreexistingData();
- ASSERT_OK_AND_ASSIGN(auto stream, fs_->OpenInputStream(data.ObjectPath()));
+ ASSERT_OK_AND_ASSIGN(auto stream, fs()->OpenInputStream(data.ObjectPath()));
ASSERT_OK(stream->Close());
std::array<char, 16> buffer{};
ASSERT_RAISES(Invalid, stream->Read(buffer.size(), buffer.data()));
@@ -1399,13 +1455,13 @@ TEST_F(TestAzuriteFileSystem, WriteMetadata) {
TEST_F(TestAzuriteFileSystem, OpenOutputStreamSmall) {
auto data = SetUpPreexistingData();
const auto path = data.ContainerPath("test-write-object");
- ASSERT_OK_AND_ASSIGN(auto output, fs_->OpenOutputStream(path, {}));
+ ASSERT_OK_AND_ASSIGN(auto output, fs()->OpenOutputStream(path, {}));
const std::string_view expected(PreexistingData::kLoremIpsum);
ASSERT_OK(output->Write(expected));
ASSERT_OK(output->Close());
// Verify we can read the object back.
- ASSERT_OK_AND_ASSIGN(auto input, fs_->OpenInputStream(path));
+ ASSERT_OK_AND_ASSIGN(auto input, fs()->OpenInputStream(path));
std::array<char, 1024> inbuf{};
ASSERT_OK_AND_ASSIGN(auto size, input->Read(inbuf.size(), inbuf.data()));
@@ -1416,7 +1472,7 @@ TEST_F(TestAzuriteFileSystem, OpenOutputStreamSmall) {
TEST_F(TestAzuriteFileSystem, OpenOutputStreamLarge) {
auto data = SetUpPreexistingData();
const auto path = data.ContainerPath("test-write-object");
- ASSERT_OK_AND_ASSIGN(auto output, fs_->OpenOutputStream(path, {}));
+ ASSERT_OK_AND_ASSIGN(auto output, fs()->OpenOutputStream(path, {}));
std::array<std::int64_t, 3> sizes{257 * 1024, 258 * 1024, 259 * 1024};
std::array<std::string, 3> buffers{
std::string(sizes[0], 'A'),
@@ -1432,7 +1488,7 @@ TEST_F(TestAzuriteFileSystem, OpenOutputStreamLarge) {
ASSERT_OK(output->Close());
// Verify we can read the object back.
- ASSERT_OK_AND_ASSIGN(auto input, fs_->OpenInputStream(path));
+ ASSERT_OK_AND_ASSIGN(auto input, fs()->OpenInputStream(path));
std::string contents;
std::shared_ptr<Buffer> buffer;
@@ -1448,26 +1504,26 @@ TEST_F(TestAzuriteFileSystem, OpenOutputStreamLarge) {
TEST_F(TestAzuriteFileSystem, OpenOutputStreamTruncatesExistingFile) {
auto data = SetUpPreexistingData();
const auto path = data.ContainerPath("test-write-object");
- ASSERT_OK_AND_ASSIGN(auto output, fs_->OpenOutputStream(path, {}));
+ ASSERT_OK_AND_ASSIGN(auto output, fs()->OpenOutputStream(path, {}));
const std::string_view expected0("Existing blob content");
ASSERT_OK(output->Write(expected0));
ASSERT_OK(output->Close());
// Check that the initial content has been written - if not this test is not
achieving
// what it's meant to.
- ASSERT_OK_AND_ASSIGN(auto input, fs_->OpenInputStream(path));
+ ASSERT_OK_AND_ASSIGN(auto input, fs()->OpenInputStream(path));
std::array<char, 1024> inbuf{};
ASSERT_OK_AND_ASSIGN(auto size, input->Read(inbuf.size(), inbuf.data()));
EXPECT_EQ(expected0, std::string_view(inbuf.data(), size));
- ASSERT_OK_AND_ASSIGN(output, fs_->OpenOutputStream(path, {}));
+ ASSERT_OK_AND_ASSIGN(output, fs()->OpenOutputStream(path, {}));
const std::string_view expected1(PreexistingData::kLoremIpsum);
ASSERT_OK(output->Write(expected1));
ASSERT_OK(output->Close());
// Verify that the initial content has been overwritten.
- ASSERT_OK_AND_ASSIGN(input, fs_->OpenInputStream(path));
+ ASSERT_OK_AND_ASSIGN(input, fs()->OpenInputStream(path));
ASSERT_OK_AND_ASSIGN(size, input->Read(inbuf.size(), inbuf.data()));
EXPECT_EQ(expected1, std::string_view(inbuf.data(), size));
}
@@ -1475,27 +1531,27 @@ TEST_F(TestAzuriteFileSystem,
OpenOutputStreamTruncatesExistingFile) {
TEST_F(TestAzuriteFileSystem, OpenAppendStreamDoesNotTruncateExistingFile) {
auto data = SetUpPreexistingData();
const auto path = data.ContainerPath("test-write-object");
- ASSERT_OK_AND_ASSIGN(auto output, fs_->OpenOutputStream(path, {}));
+ ASSERT_OK_AND_ASSIGN(auto output, fs()->OpenOutputStream(path, {}));
const std::string_view expected0("Existing blob content");
ASSERT_OK(output->Write(expected0));
ASSERT_OK(output->Close());
// Check that the initial content has been written - if not this test is not
achieving
// what it's meant to.
- ASSERT_OK_AND_ASSIGN(auto input, fs_->OpenInputStream(path));
+ ASSERT_OK_AND_ASSIGN(auto input, fs()->OpenInputStream(path));
std::array<char, 1024> inbuf{};
ASSERT_OK_AND_ASSIGN(auto size, input->Read(inbuf.size(), inbuf.data()));
EXPECT_EQ(expected0, std::string_view(inbuf.data()));
- ASSERT_OK_AND_ASSIGN(output, fs_->OpenAppendStream(path, {}));
+ ASSERT_OK_AND_ASSIGN(output, fs()->OpenAppendStream(path, {}));
const std::string_view expected1(PreexistingData::kLoremIpsum);
ASSERT_OK(output->Write(expected1));
ASSERT_OK(output->Close());
// Verify that the initial content has not been overwritten and that the
block from
// the other client was not committed.
- ASSERT_OK_AND_ASSIGN(input, fs_->OpenInputStream(path));
+ ASSERT_OK_AND_ASSIGN(input, fs()->OpenInputStream(path));
ASSERT_OK_AND_ASSIGN(size, input->Read(inbuf.size(), inbuf.data()));
EXPECT_EQ(std::string(inbuf.data(), size),
std::string(expected0) + std::string(expected1));
@@ -1504,7 +1560,7 @@ TEST_F(TestAzuriteFileSystem,
OpenAppendStreamDoesNotTruncateExistingFile) {
TEST_F(TestAzuriteFileSystem, OpenOutputStreamClosed) {
auto data = SetUpPreexistingData();
const auto path = data.ContainerPath("open-output-stream-closed.txt");
- ASSERT_OK_AND_ASSIGN(auto output, fs_->OpenOutputStream(path, {}));
+ ASSERT_OK_AND_ASSIGN(auto output, fs()->OpenOutputStream(path, {}));
ASSERT_OK(output->Close());
ASSERT_RAISES(Invalid, output->Write(PreexistingData::kLoremIpsum,
std::strlen(PreexistingData::kLoremIpsum)));
@@ -1515,7 +1571,7 @@ TEST_F(TestAzuriteFileSystem, OpenOutputStreamClosed) {
TEST_F(TestAzuriteFileSystem, OpenOutputStreamUri) {
auto data = SetUpPreexistingData();
const auto path = data.ContainerPath("open-output-stream-uri.txt");
- ASSERT_RAISES(Invalid, fs_->OpenInputStream("abfs://" + path));
+ ASSERT_RAISES(Invalid, fs()->OpenInputStream("abfs://" + path));
}
TEST_F(TestAzuriteFileSystem, OpenInputFileMixedReadVsReadAt) {
@@ -1534,7 +1590,7 @@ TEST_F(TestAzuriteFileSystem,
OpenInputFileMixedReadVsReadAt) {
UploadLines(lines, path, kLineCount * kLineWidth);
std::shared_ptr<io::RandomAccessFile> file;
- ASSERT_OK_AND_ASSIGN(file, fs_->OpenInputFile(path));
+ ASSERT_OK_AND_ASSIGN(file, fs()->OpenInputFile(path));
for (int i = 0; i != 32; ++i) {
SCOPED_TRACE("Iteration " + std::to_string(i));
// Verify sequential reads work as expected.
@@ -1582,7 +1638,7 @@ TEST_F(TestAzuriteFileSystem, OpenInputFileRandomSeek) {
UploadLines(lines, path, kLineCount * kLineWidth);
std::shared_ptr<io::RandomAccessFile> file;
- ASSERT_OK_AND_ASSIGN(file, fs_->OpenInputFile(path));
+ ASSERT_OK_AND_ASSIGN(file, fs()->OpenInputFile(path));
for (int i = 0; i != 32; ++i) {
SCOPED_TRACE("Iteration " + std::to_string(i));
// Verify sequential reads work as expected.
@@ -1607,16 +1663,16 @@ TEST_F(TestAzuriteFileSystem, OpenInputFileIoContext) {
contents.length());
std::shared_ptr<io::RandomAccessFile> file;
- ASSERT_OK_AND_ASSIGN(file, fs_->OpenInputFile(path));
- EXPECT_EQ(fs_->io_context().external_id(), file->io_context().external_id());
+ ASSERT_OK_AND_ASSIGN(file, fs()->OpenInputFile(path));
+ EXPECT_EQ(fs()->io_context().external_id(),
file->io_context().external_id());
}
TEST_F(TestAzuriteFileSystem, OpenInputFileInfo) {
auto data = SetUpPreexistingData();
- ASSERT_OK_AND_ASSIGN(auto info, fs_->GetFileInfo(data.ObjectPath()));
+ ASSERT_OK_AND_ASSIGN(auto info, fs()->GetFileInfo(data.ObjectPath()));
std::shared_ptr<io::RandomAccessFile> file;
- ASSERT_OK_AND_ASSIGN(file, fs_->OpenInputFile(info));
+ ASSERT_OK_AND_ASSIGN(file, fs()->OpenInputFile(info));
std::array<char, 1024> buffer{};
std::int64_t size;
@@ -1629,21 +1685,21 @@ TEST_F(TestAzuriteFileSystem, OpenInputFileInfo) {
TEST_F(TestAzuriteFileSystem, OpenInputFileNotFound) {
auto data = SetUpPreexistingData();
- ASSERT_RAISES(IOError, fs_->OpenInputFile(data.NotFoundObjectPath()));
+ ASSERT_RAISES(IOError, fs()->OpenInputFile(data.NotFoundObjectPath()));
}
TEST_F(TestAzuriteFileSystem, OpenInputFileInfoInvalid) {
auto data = SetUpPreexistingData();
- ASSERT_OK_AND_ASSIGN(auto info, fs_->GetFileInfo(data.container_name));
- ASSERT_RAISES(IOError, fs_->OpenInputFile(info));
+ ASSERT_OK_AND_ASSIGN(auto info, fs()->GetFileInfo(data.container_name));
+ ASSERT_RAISES(IOError, fs()->OpenInputFile(info));
- ASSERT_OK_AND_ASSIGN(auto info2,
fs_->GetFileInfo(data.NotFoundObjectPath()));
- ASSERT_RAISES(IOError, fs_->OpenInputFile(info2));
+ ASSERT_OK_AND_ASSIGN(auto info2,
fs()->GetFileInfo(data.NotFoundObjectPath()));
+ ASSERT_RAISES(IOError, fs()->OpenInputFile(info2));
}
TEST_F(TestAzuriteFileSystem, OpenInputFileClosed) {
auto data = SetUpPreexistingData();
- ASSERT_OK_AND_ASSIGN(auto stream, fs_->OpenInputFile(data.ObjectPath()));
+ ASSERT_OK_AND_ASSIGN(auto stream, fs()->OpenInputFile(data.ObjectPath()));
ASSERT_OK(stream->Close());
std::array<char, 16> buffer{};
ASSERT_RAISES(Invalid, stream->Tell());
@@ -1654,6 +1710,5 @@ TEST_F(TestAzuriteFileSystem, OpenInputFileClosed) {
ASSERT_RAISES(Invalid, stream->Seek(2));
}
-} // namespace
} // namespace fs
} // namespace arrow
