marin-ma commented on code in PR #7861:
URL: https://github.com/apache/incubator-gluten/pull/7861#discussion_r1846483470
##########
cpp/core/shuffle/Utils.h:
##########
@@ -72,4 +72,37 @@ arrow::Result<std::shared_ptr<arrow::RecordBatch>>
makeUncompressedRecordBatch(
std::shared_ptr<arrow::Buffer> zeroLengthNullBuffer();
+// MmapFileStream is used to optimize sequential file reading. It uses madvise
+// to prefetch and release memory timely.
+class MmapFileStream : public arrow::io::InputStream {
+ public:
+ MmapFileStream(arrow::internal::FileDescriptor fd, uint8_t* data, int64_t
size)
Review Comment:
Please separate the declaration and definition. And add a blank line between
two member functions.
##########
cpp/core/shuffle/Utils.h:
##########
@@ -72,4 +72,37 @@ arrow::Result<std::shared_ptr<arrow::RecordBatch>>
makeUncompressedRecordBatch(
std::shared_ptr<arrow::Buffer> zeroLengthNullBuffer();
+// MmapFileStream is used to optimize sequential file reading. It uses madvise
+// to prefetch and release memory timely.
+class MmapFileStream : public arrow::io::InputStream {
+ public:
+ MmapFileStream(arrow::internal::FileDescriptor fd, uint8_t* data, int64_t
size)
+ : fd_(std::move(fd)), data_(data), size_(size){};
+ static arrow::Result<std::shared_ptr<MmapFileStream>> open(const
std::string& path);
+ arrow::Result<int64_t> Tell() const override;
+ arrow::Status Close() override;
+ arrow::Result<int64_t> Read(int64_t nbytes, void* out) override;
+ arrow::Result<std::shared_ptr<arrow::Buffer>> Read(int64_t nbytes) override;
+ bool closed() const override {
Review Comment:
ditto
##########
cpp/core/shuffle/Utils.h:
##########
@@ -72,4 +72,37 @@ arrow::Result<std::shared_ptr<arrow::RecordBatch>>
makeUncompressedRecordBatch(
std::shared_ptr<arrow::Buffer> zeroLengthNullBuffer();
+// MmapFileStream is used to optimize sequential file reading. It uses madvise
+// to prefetch and release memory timely.
+class MmapFileStream : public arrow::io::InputStream {
+ public:
+ MmapFileStream(arrow::internal::FileDescriptor fd, uint8_t* data, int64_t
size)
+ : fd_(std::move(fd)), data_(data), size_(size){};
+ static arrow::Result<std::shared_ptr<MmapFileStream>> open(const
std::string& path);
+ arrow::Result<int64_t> Tell() const override;
+ arrow::Status Close() override;
+ arrow::Result<int64_t> Read(int64_t nbytes, void* out) override;
+ arrow::Result<std::shared_ptr<arrow::Buffer>> Read(int64_t nbytes) override;
+ bool closed() const override {
+ return data_ == nullptr;
+ };
+
+ private:
+ arrow::Result<int64_t> actualReadSize(int64_t nbytes) {
Review Comment:
ditto
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
