icexelloss commented on issue #37630:
URL: https://github.com/apache/arrow/issues/37630#issuecomment-1737559313
FWIW, This is my local patch:
```
diff --git a/c/src/arrow/dataset/dataset.h b/c/src/arrow/dataset/dataset.h
index
1db230b16e9c2e52ad58c02255f87375307271d0..37c2ddca4884d32bb1721dd3a2d937f9faf6b93f
100644
--- a/c/src/arrow/dataset/dataset.h
+++ b/c/src/arrow/dataset/dataset.h
@@ -199,6 +199,9 @@ class ARROW_DS_EXPORT Fragment : public
std::enable_shared_from_this<Fragment> {
return partition_expression_;
}
+ /// \brief Clear metadata in the fragment. By default this is an noop.
+ virtual void ClearMetadata() { return; }
+
virtual ~Fragment() = default;
protected:
diff --git a/c/src/arrow/dataset/file_parquet.h
b/c/src/arrow/dataset/file_parquet.h
index
f33190bd93347f781df91a6cda612031f09caf75..632a555894a7b91ee6ac722a6820a9eaff61f232
100644
--- a/c/src/arrow/dataset/file_parquet.h
+++ b/c/src/arrow/dataset/file_parquet.h
@@ -174,6 +174,11 @@ class ARROW_DS_EXPORT ParquetFileFragment : public
FileFragment {
static std::optional<compute::Expression> EvaluateStatisticsAsExpression(
const Field& field, const parquet::Statistics& statistics);
+ void ClearMetadata() override {
+ metadata_ = NULLPTR;
+ manifest_ = NULLPTR;
+ }
+
private:
ParquetFileFragment(FileSource source, std::shared_ptr<FileFormat> format,
compute::Expression partition_expression,
diff --git a/c/src/arrow/dataset/scanner.cc b/c/src/arrow/dataset/scanner.cc
index
18981d1451980f4b775fb87e42dcaede89a1fc7c..8f83a7c2dc52b8f5c728b91be2f93442168002d7
100644
--- a/c/src/arrow/dataset/scanner.cc
+++ b/c/src/arrow/dataset/scanner.cc
@@ -1037,6 +1037,9 @@ Result<acero::ExecNode*> MakeScanNode(acero::ExecPlan*
plan,
// unnecessarily materialized columns in batch. We could drop them
now instead of
// letting them coast through the rest of the plan.
auto guarantee = partial.fragment.value->partition_expression();
+ // This is safe because by the time we reach here we have already
decoded
+ // the fragment into a batch so the metadata is no longer needed.
+ partial.fragment.value->ClearMetadata();
ARROW_ASSIGN_OR_RAISE(
std::optional<compute::ExecBatch> batch,
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
