This is an automated email from the ASF dual-hosted git repository.
felixybw pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 403be3977 [VL] bug fix for S3 read (#6313)
403be3977 is described below
commit 403be39775d0042d17ac27e3a8a714016be88400
Author: BInwei Yang <[email protected]>
AuthorDate: Tue Jul 2 18:37:43 2024 -0700
[VL] bug fix for S3 read (#6313)
bugfix: Velox s3FileSystem use -1 as default file size. Gluten wrongly
passed 0 if file size isn't gotten from Spark.
---
cpp/velox/compute/VeloxPlanConverter.cc | 7 ++++++-
1 file changed, 6 insertions(+), 1 deletion(-)
diff --git a/cpp/velox/compute/VeloxPlanConverter.cc
b/cpp/velox/compute/VeloxPlanConverter.cc
index bcd03b110..315ff2da6 100644
--- a/cpp/velox/compute/VeloxPlanConverter.cc
+++ b/cpp/velox/compute/VeloxPlanConverter.cc
@@ -81,7 +81,12 @@ std::shared_ptr<SplitInfo> parseScanSplitInfo(
splitInfo->paths.emplace_back(file.uri_file());
splitInfo->starts.emplace_back(file.start());
splitInfo->lengths.emplace_back(file.length());
- facebook::velox::FileProperties fileProps = {file.properties().filesize(),
file.properties().modificationtime()};
+
+ facebook::velox::FileProperties fileProps;
+ if (file.has_properties()) {
+ fileProps.fileSize = file.properties().filesize();
+ fileProps.modificationTime = file.properties().modificationtime();
+ }
splitInfo->properties.emplace_back(fileProps);
switch (file.file_format_case()) {
case SubstraitFileFormatCase::kOrc:
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
