westonpace commented on code in PR #36779:
URL: https://github.com/apache/arrow/pull/36779#discussion_r1296270804
##########
cpp/src/parquet/arrow/arrow_reader_writer_test.cc:
##########
@@ -3963,6 +4096,88 @@ TEST(TestArrowReaderAdHoc,
LARGE_MEMORY_TEST(LargeStringColumn)) {
AssertTablesEqual(*table, *batched_table, /*same_chunk_layout=*/false);
}
+TEST(TestArrowReaderAdHoc, LARGE_MEMORY_TEST(LargeStringValue)) {
+ // ARROW-3762
+ ::arrow::StringBuilder builder;
+ // 16 rows of 256MiB bytes each is 4GiB. This will get put into
+ // 3 chunks of 7 rows, 7 rows, and 2 rows.
+ constexpr std::int32_t kValueSize = 1 << 28;
+ constexpr std::int32_t kNumRows = 4;
+ constexpr std::int32_t kNumChunks = 4;
+ std::vector<std::shared_ptr<Array>> chunks;
+ std::vector<uint8_t> value(kValueSize, '0');
+ for (int chunk_idx = 0; chunk_idx < kNumChunks; chunk_idx++) {
+ ASSERT_OK(builder.Resize(kNumRows));
+ ASSERT_OK(builder.ReserveData(kNumRows * kValueSize));
+ for (int64_t i = 0; i < kNumRows; ++i) {
+ builder.UnsafeAppend(value.data(), kValueSize);
+ }
+ std::shared_ptr<Array> array;
+ ASSERT_OK(builder.Finish(&array));
+ chunks.push_back(std::move(array));
+ }
+
+ // Eaglerly free up memory
+ value.clear();
Review Comment:
I put `value` in its own scope block.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
