realAaronWu commented on code in PR #586: URL: https://github.com/apache/paimon-cpp/pull/586#discussion_r4656892250
########## src/paimon/common/utils/arrow/arrow_lifetime_failure_test.cpp: ########## @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include <cstddef> +#include <memory> +#include <new> +#include <utility> + +#include "gtest/gtest.h" +#include "paimon/arrow/abi.h" +#include "paimon/common/utils/arrow/mem_utils.h" + +namespace { +thread_local bool fail_next_allocation = false; +} // namespace + +extern "C" void* __real__Znwm(std::size_t size); +extern "C" void* __wrap__Znwm(std::size_t size) { + if (std::exchange(fail_next_allocation, false)) { + throw std::bad_alloc(); + } + return __real__Znwm(size); +} + +namespace paimon::test { + +TEST(ArrowLifetimeFailureTest, ReleasesBothRootsWhenHoldingLifetimeRunsOutOfMemory) { + ArrowArray array{}; + ArrowSchema schema{}; + int array_releases = 0; + int schema_releases = 0; + array.private_data = &array_releases; + array.release = [](ArrowArray* root) { + ++*static_cast<int*>(root->private_data); + root->release = nullptr; + }; + schema.private_data = &schema_releases; + schema.release = [](ArrowSchema* root) { + ++*static_cast<int*>(root->private_data); + root->release = nullptr; + }; + std::shared_ptr<void> owner = std::make_shared<int>(1); + Review Comment: Updated the release counters, their callback casts, and the test owner to use `int32_t` in 4d50bc42. The allocation-failure test passed on Linux ARM64 and observed one release for each Arrow root. ########## src/paimon/format/parquet/parquet_file_batch_reader_test.cpp: ########## @@ -667,6 +668,56 @@ TEST_F(ParquetFileBatchReaderTest, TestNextBatchSimple) { } } +TEST_F(ParquetFileBatchReaderTest, ReleasesUnpublishedBatchWhenMetricsFail) { + class FailingReadMetrics final : public MetricsImpl { + public: + void SetCounter(const std::string& name, uint64_t value) override { + if (name == ParquetMetrics::READ_ROWS) { + failed = true; + throw std::bad_alloc(); + } + MetricsImpl::SetCounter(name, value); + } + + bool failed = false; + }; + + const auto file_name = paimon::test::GetDataDir() + + "parquet/parquet_append_table.db/parquet_append_table/bucket-0/" + "data-9ea62f34-1dca-49c1-bf7a-d37303d8fb76-0.parquet"; + const auto initial_bytes = pool_->bytes_allocated(); + const auto initial_export_bytes = arrow::default_memory_pool()->bytes_allocated(); + auto metrics = std::make_shared<FailingReadMetrics>(); + auto reader = + PrepareParquetFileBatchReader(file_name, schema_, /*predicate=*/nullptr, + /*selection_bitmap=*/std::nullopt, /*batch_size=*/1); + ASSERT_TRUE(reader); + reader->metrics_ = metrics; + const auto batch = reader->NextBatch(); + EXPECT_TRUE(metrics->failed); + ASSERT_FALSE(batch.ok()); + EXPECT_TRUE(batch.status().IsInvalid()); Review Comment: This first `NextBatch()` deliberately fails after Arrow export. In 4d50bc42 it runs once, and `ASSERT_NOK(batch)` checks the failure before we verify `IsInvalid()` and both memory pools. The successful reopen already uses `ASSERT_OK_AND_ASSIGN` and verifies a one-row batch. The regression passed on Linux ARM64. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
