coryan commented on a change in pull request #11812:
URL: https://github.com/apache/arrow/pull/11812#discussion_r760230047
##########
File path: cpp/src/arrow/filesystem/gcsfs_test.cc
##########
@@ -119,7 +122,23 @@ class GcsIntegrationTest : public ::testing::Test {
.set<gc::UnifiedCredentialsOption>(gc::MakeInsecureCredentials()));
}
+ std::string RandomLine(int lineno, std::size_t width) {
+ auto const fillers = std::string("abcdefghijlkmnopqrstuvwxyz0123456789");
+ std::uniform_int_distribution<std::size_t> d(0, fillers.size() - 1);
Review comment:
`random_string()` produces non-printable characters, which are not the
easiest thing to troubleshoot. `random_ascii()` could do it, but has a weird
API using a `uint8_t*` output parameter. I would rather leave this as-is.
##########
File path: cpp/src/arrow/filesystem/gcsfs_test.cc
##########
@@ -521,6 +540,130 @@ TEST_F(GcsIntegrationTest, WriteObjectLarge) {
EXPECT_EQ(contents, buffers[0] + buffers[1] + buffers[2]);
}
+TEST_F(GcsIntegrationTest, OpenInputFileMixedReadVsReadAt) {
+ auto fs = internal::MakeGcsFileSystemForTest(TestGcsOptions());
+
+ // Create a file large enough to make the random access tests non-trivial.
+ auto constexpr kLineWidth = 100;
+ auto constexpr kLineCount = 4096;
+ std::vector<std::string> lines(kLineCount);
+ int lineno = 0;
+ std::generate_n(lines.begin(), lines.size(),
+ [&] { return RandomLine(++lineno, kLineWidth); });
+
+ const auto path =
+ kPreexistingBucket +
std::string("/OpenInputFileMixedReadVsReadAt/object-name");
+ std::shared_ptr<io::OutputStream> output;
+ ASSERT_OK_AND_ASSIGN(output, fs->OpenOutputStream(path, {}));
+ for (auto const& line : lines) {
+ ASSERT_OK(output->Write(line.data(), line.size()));
+ }
+ ASSERT_OK(output->Close());
+
+ std::shared_ptr<io::RandomAccessFile> file;
+ ASSERT_OK_AND_ASSIGN(file, fs->OpenInputFile(path));
+ for (int i = 0; i != 32; ++i) {
+ SCOPED_TRACE("Iteration " + std::to_string(i));
+ // Verify sequential reads work as expected.
+ std::array<char, kLineWidth> buffer{};
+ std::int64_t size;
+ {
+ ASSERT_OK_AND_ASSIGN(auto actual, file->Read(kLineWidth));
+ EXPECT_EQ(lines[2 * i], actual->ToString());
+ }
+ {
+ ASSERT_OK_AND_ASSIGN(size, file->Read(buffer.size(), buffer.data()));
+ EXPECT_EQ(size, kLineWidth);
+ auto actual = std::string{buffer.begin(), buffer.end()};
+ EXPECT_EQ(lines[2 * i + 1], actual);
+ }
+
+ // Verify random reads interleave too.
+ auto const index = RandomIndex(kLineCount);
+ auto const position = index * kLineWidth;
+ ASSERT_OK_AND_ASSIGN(size, file->ReadAt(position, buffer.size(),
buffer.data()));
+ EXPECT_EQ(size, kLineWidth);
+ auto actual = std::string{buffer.begin(), buffer.end()};
+ EXPECT_EQ(lines[index], actual);
+
+ // Verify random reads using buffers work.
+ ASSERT_OK_AND_ASSIGN(auto b, file->ReadAt(position, kLineWidth));
+ EXPECT_EQ(lines[index], b->ToString());
+ }
+}
+
+TEST_F(GcsIntegrationTest, OpenInputFileRandomSeek) {
+ auto fs = internal::MakeGcsFileSystemForTest(TestGcsOptions());
+
+ // Create a file large enough to make the random access tests non-trivial.
+ auto constexpr kLineWidth = 100;
+ auto constexpr kLineCount = 4096;
+ std::vector<std::string> lines(kLineCount);
+ int lineno = 0;
+ std::generate_n(lines.begin(), lines.size(),
+ [&] { return RandomLine(++lineno, kLineWidth); });
+
+ const auto path =
+ kPreexistingBucket + std::string("/OpenInputFileRandomSeek/object-name");
+ std::shared_ptr<io::OutputStream> output;
+ ASSERT_OK_AND_ASSIGN(output, fs->OpenOutputStream(path, {}));
+ for (auto const& line : lines) {
+ ASSERT_OK(output->Write(line.data(), line.size()));
+ }
+ ASSERT_OK(output->Close());
+
+ std::shared_ptr<io::RandomAccessFile> file;
+ ASSERT_OK_AND_ASSIGN(file, fs->OpenInputFile(path));
+ for (int i = 0; i != 32; ++i) {
+ SCOPED_TRACE("Iteration " + std::to_string(i));
+ // Verify sequential reads work as expected.
+ auto const index = RandomIndex(kLineCount);
+ auto const position = index * kLineWidth;
+ ASSERT_OK(file->Seek(position));
+ ASSERT_OK_AND_ASSIGN(auto actual, file->Read(kLineWidth));
+ EXPECT_EQ(lines[index], actual->ToString());
+ }
+}
+
+TEST_F(GcsIntegrationTest, OpenInputFileInfo) {
+ auto fs = internal::MakeGcsFileSystemForTest(TestGcsOptions());
+
+ arrow::fs::FileInfo info;
+ ASSERT_OK_AND_ASSIGN(info, fs->GetFileInfo(PreexistingObjectPath()));
+
+ std::shared_ptr<io::RandomAccessFile> file;
+ ASSERT_OK_AND_ASSIGN(file, fs->OpenInputFile(info));
+
+ std::array<char, 1024> buffer{};
+ std::int64_t size;
+ auto constexpr kStart = 16;
+ ASSERT_OK_AND_ASSIGN(size, file->ReadAt(kStart, buffer.size(),
buffer.data()));
+
+ auto const expected = std::string(kLoremIpsum).substr(kStart);
+ EXPECT_EQ(std::string(buffer.data(), size), expected);
+}
+
+TEST_F(GcsIntegrationTest, OpenInputFileNotFound) {
+ auto fs = internal::MakeGcsFileSystemForTest(TestGcsOptions());
+
+ auto result = fs->OpenInputFile(NotFoundObjectPath());
+ EXPECT_EQ(result.status().code(), StatusCode::IOError);
Review comment:
Done.
##########
File path: cpp/src/arrow/filesystem/gcsfs_test.cc
##########
@@ -521,6 +540,130 @@ TEST_F(GcsIntegrationTest, WriteObjectLarge) {
EXPECT_EQ(contents, buffers[0] + buffers[1] + buffers[2]);
}
+TEST_F(GcsIntegrationTest, OpenInputFileMixedReadVsReadAt) {
+ auto fs = internal::MakeGcsFileSystemForTest(TestGcsOptions());
+
+ // Create a file large enough to make the random access tests non-trivial.
+ auto constexpr kLineWidth = 100;
+ auto constexpr kLineCount = 4096;
+ std::vector<std::string> lines(kLineCount);
+ int lineno = 0;
+ std::generate_n(lines.begin(), lines.size(),
+ [&] { return RandomLine(++lineno, kLineWidth); });
+
+ const auto path =
+ kPreexistingBucket +
std::string("/OpenInputFileMixedReadVsReadAt/object-name");
+ std::shared_ptr<io::OutputStream> output;
+ ASSERT_OK_AND_ASSIGN(output, fs->OpenOutputStream(path, {}));
+ for (auto const& line : lines) {
+ ASSERT_OK(output->Write(line.data(), line.size()));
+ }
+ ASSERT_OK(output->Close());
+
+ std::shared_ptr<io::RandomAccessFile> file;
+ ASSERT_OK_AND_ASSIGN(file, fs->OpenInputFile(path));
+ for (int i = 0; i != 32; ++i) {
+ SCOPED_TRACE("Iteration " + std::to_string(i));
+ // Verify sequential reads work as expected.
+ std::array<char, kLineWidth> buffer{};
+ std::int64_t size;
+ {
+ ASSERT_OK_AND_ASSIGN(auto actual, file->Read(kLineWidth));
+ EXPECT_EQ(lines[2 * i], actual->ToString());
+ }
+ {
+ ASSERT_OK_AND_ASSIGN(size, file->Read(buffer.size(), buffer.data()));
+ EXPECT_EQ(size, kLineWidth);
+ auto actual = std::string{buffer.begin(), buffer.end()};
+ EXPECT_EQ(lines[2 * i + 1], actual);
+ }
+
+ // Verify random reads interleave too.
+ auto const index = RandomIndex(kLineCount);
+ auto const position = index * kLineWidth;
+ ASSERT_OK_AND_ASSIGN(size, file->ReadAt(position, buffer.size(),
buffer.data()));
+ EXPECT_EQ(size, kLineWidth);
+ auto actual = std::string{buffer.begin(), buffer.end()};
+ EXPECT_EQ(lines[index], actual);
+
+ // Verify random reads using buffers work.
+ ASSERT_OK_AND_ASSIGN(auto b, file->ReadAt(position, kLineWidth));
+ EXPECT_EQ(lines[index], b->ToString());
+ }
+}
+
+TEST_F(GcsIntegrationTest, OpenInputFileRandomSeek) {
+ auto fs = internal::MakeGcsFileSystemForTest(TestGcsOptions());
+
+ // Create a file large enough to make the random access tests non-trivial.
+ auto constexpr kLineWidth = 100;
+ auto constexpr kLineCount = 4096;
+ std::vector<std::string> lines(kLineCount);
+ int lineno = 0;
+ std::generate_n(lines.begin(), lines.size(),
+ [&] { return RandomLine(++lineno, kLineWidth); });
+
+ const auto path =
+ kPreexistingBucket + std::string("/OpenInputFileRandomSeek/object-name");
+ std::shared_ptr<io::OutputStream> output;
+ ASSERT_OK_AND_ASSIGN(output, fs->OpenOutputStream(path, {}));
+ for (auto const& line : lines) {
+ ASSERT_OK(output->Write(line.data(), line.size()));
+ }
+ ASSERT_OK(output->Close());
+
+ std::shared_ptr<io::RandomAccessFile> file;
+ ASSERT_OK_AND_ASSIGN(file, fs->OpenInputFile(path));
+ for (int i = 0; i != 32; ++i) {
+ SCOPED_TRACE("Iteration " + std::to_string(i));
+ // Verify sequential reads work as expected.
+ auto const index = RandomIndex(kLineCount);
+ auto const position = index * kLineWidth;
+ ASSERT_OK(file->Seek(position));
+ ASSERT_OK_AND_ASSIGN(auto actual, file->Read(kLineWidth));
+ EXPECT_EQ(lines[index], actual->ToString());
+ }
+}
+
+TEST_F(GcsIntegrationTest, OpenInputFileInfo) {
+ auto fs = internal::MakeGcsFileSystemForTest(TestGcsOptions());
+
+ arrow::fs::FileInfo info;
+ ASSERT_OK_AND_ASSIGN(info, fs->GetFileInfo(PreexistingObjectPath()));
+
+ std::shared_ptr<io::RandomAccessFile> file;
+ ASSERT_OK_AND_ASSIGN(file, fs->OpenInputFile(info));
+
+ std::array<char, 1024> buffer{};
+ std::int64_t size;
+ auto constexpr kStart = 16;
+ ASSERT_OK_AND_ASSIGN(size, file->ReadAt(kStart, buffer.size(),
buffer.data()));
+
+ auto const expected = std::string(kLoremIpsum).substr(kStart);
+ EXPECT_EQ(std::string(buffer.data(), size), expected);
+}
+
+TEST_F(GcsIntegrationTest, OpenInputFileNotFound) {
+ auto fs = internal::MakeGcsFileSystemForTest(TestGcsOptions());
+
+ auto result = fs->OpenInputFile(NotFoundObjectPath());
+ EXPECT_EQ(result.status().code(), StatusCode::IOError);
+}
+
+TEST_F(GcsIntegrationTest, OpenInputFileInfoInvalid) {
+ auto fs = internal::MakeGcsFileSystemForTest(TestGcsOptions());
+
+ arrow::fs::FileInfo info;
+ ASSERT_OK_AND_ASSIGN(info, fs->GetFileInfo(kPreexistingBucket));
+
+ auto result = fs->OpenInputFile(NotFoundObjectPath());
+ EXPECT_EQ(result.status().code(), StatusCode::IOError);
+
+ ASSERT_OK_AND_ASSIGN(info, fs->GetFileInfo(NotFoundObjectPath()));
+ result = fs->OpenInputFile(NotFoundObjectPath());
Review comment:
Yes, thanks. Fixed.
##########
File path: cpp/src/arrow/filesystem/gcsfs_test.cc
##########
@@ -521,6 +540,130 @@ TEST_F(GcsIntegrationTest, WriteObjectLarge) {
EXPECT_EQ(contents, buffers[0] + buffers[1] + buffers[2]);
}
+TEST_F(GcsIntegrationTest, OpenInputFileMixedReadVsReadAt) {
+ auto fs = internal::MakeGcsFileSystemForTest(TestGcsOptions());
+
+ // Create a file large enough to make the random access tests non-trivial.
+ auto constexpr kLineWidth = 100;
+ auto constexpr kLineCount = 4096;
+ std::vector<std::string> lines(kLineCount);
+ int lineno = 0;
+ std::generate_n(lines.begin(), lines.size(),
+ [&] { return RandomLine(++lineno, kLineWidth); });
+
+ const auto path =
+ kPreexistingBucket +
std::string("/OpenInputFileMixedReadVsReadAt/object-name");
+ std::shared_ptr<io::OutputStream> output;
+ ASSERT_OK_AND_ASSIGN(output, fs->OpenOutputStream(path, {}));
+ for (auto const& line : lines) {
+ ASSERT_OK(output->Write(line.data(), line.size()));
+ }
+ ASSERT_OK(output->Close());
+
+ std::shared_ptr<io::RandomAccessFile> file;
+ ASSERT_OK_AND_ASSIGN(file, fs->OpenInputFile(path));
+ for (int i = 0; i != 32; ++i) {
+ SCOPED_TRACE("Iteration " + std::to_string(i));
+ // Verify sequential reads work as expected.
+ std::array<char, kLineWidth> buffer{};
+ std::int64_t size;
+ {
+ ASSERT_OK_AND_ASSIGN(auto actual, file->Read(kLineWidth));
+ EXPECT_EQ(lines[2 * i], actual->ToString());
+ }
+ {
+ ASSERT_OK_AND_ASSIGN(size, file->Read(buffer.size(), buffer.data()));
+ EXPECT_EQ(size, kLineWidth);
+ auto actual = std::string{buffer.begin(), buffer.end()};
+ EXPECT_EQ(lines[2 * i + 1], actual);
+ }
+
+ // Verify random reads interleave too.
+ auto const index = RandomIndex(kLineCount);
+ auto const position = index * kLineWidth;
+ ASSERT_OK_AND_ASSIGN(size, file->ReadAt(position, buffer.size(),
buffer.data()));
+ EXPECT_EQ(size, kLineWidth);
+ auto actual = std::string{buffer.begin(), buffer.end()};
+ EXPECT_EQ(lines[index], actual);
+
+ // Verify random reads using buffers work.
+ ASSERT_OK_AND_ASSIGN(auto b, file->ReadAt(position, kLineWidth));
+ EXPECT_EQ(lines[index], b->ToString());
+ }
+}
+
+TEST_F(GcsIntegrationTest, OpenInputFileRandomSeek) {
+ auto fs = internal::MakeGcsFileSystemForTest(TestGcsOptions());
+
+ // Create a file large enough to make the random access tests non-trivial.
+ auto constexpr kLineWidth = 100;
+ auto constexpr kLineCount = 4096;
+ std::vector<std::string> lines(kLineCount);
+ int lineno = 0;
+ std::generate_n(lines.begin(), lines.size(),
+ [&] { return RandomLine(++lineno, kLineWidth); });
+
+ const auto path =
+ kPreexistingBucket + std::string("/OpenInputFileRandomSeek/object-name");
+ std::shared_ptr<io::OutputStream> output;
+ ASSERT_OK_AND_ASSIGN(output, fs->OpenOutputStream(path, {}));
+ for (auto const& line : lines) {
+ ASSERT_OK(output->Write(line.data(), line.size()));
+ }
+ ASSERT_OK(output->Close());
+
+ std::shared_ptr<io::RandomAccessFile> file;
+ ASSERT_OK_AND_ASSIGN(file, fs->OpenInputFile(path));
+ for (int i = 0; i != 32; ++i) {
+ SCOPED_TRACE("Iteration " + std::to_string(i));
+ // Verify sequential reads work as expected.
+ auto const index = RandomIndex(kLineCount);
+ auto const position = index * kLineWidth;
+ ASSERT_OK(file->Seek(position));
+ ASSERT_OK_AND_ASSIGN(auto actual, file->Read(kLineWidth));
+ EXPECT_EQ(lines[index], actual->ToString());
+ }
+}
+
+TEST_F(GcsIntegrationTest, OpenInputFileInfo) {
+ auto fs = internal::MakeGcsFileSystemForTest(TestGcsOptions());
+
+ arrow::fs::FileInfo info;
+ ASSERT_OK_AND_ASSIGN(info, fs->GetFileInfo(PreexistingObjectPath()));
+
+ std::shared_ptr<io::RandomAccessFile> file;
+ ASSERT_OK_AND_ASSIGN(file, fs->OpenInputFile(info));
+
+ std::array<char, 1024> buffer{};
+ std::int64_t size;
+ auto constexpr kStart = 16;
+ ASSERT_OK_AND_ASSIGN(size, file->ReadAt(kStart, buffer.size(),
buffer.data()));
+
+ auto const expected = std::string(kLoremIpsum).substr(kStart);
+ EXPECT_EQ(std::string(buffer.data(), size), expected);
+}
+
+TEST_F(GcsIntegrationTest, OpenInputFileNotFound) {
+ auto fs = internal::MakeGcsFileSystemForTest(TestGcsOptions());
+
+ auto result = fs->OpenInputFile(NotFoundObjectPath());
+ EXPECT_EQ(result.status().code(), StatusCode::IOError);
+}
+
+TEST_F(GcsIntegrationTest, OpenInputFileInfoInvalid) {
+ auto fs = internal::MakeGcsFileSystemForTest(TestGcsOptions());
+
+ arrow::fs::FileInfo info;
+ ASSERT_OK_AND_ASSIGN(info, fs->GetFileInfo(kPreexistingBucket));
+
+ auto result = fs->OpenInputFile(NotFoundObjectPath());
Review comment:
Yup, fixed too.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
