This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-1.2-lts by this push:
new f777d3b36a [branch1.2](hudi) skip hidden file of hudi
f777d3b36a is described below
commit f777d3b36af16564f1252b97c5a4d855ca01d20d
Author: morningman <[email protected]>
AuthorDate: Mon May 22 16:15:55 2023 +0800
[branch1.2](hudi) skip hidden file of hudi
When listing data files from hudi's data dir, there are some hidden files
such as `.hoodie_metadata`, we should skip these files.
Otherwise, Doris will read these file as data file, and return file format
error
---
.../doris/planner/external/HudiScanProvider.java | 21 +++++++++++++++++++++
1 file changed, 21 insertions(+)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/planner/external/HudiScanProvider.java
b/fe/fe-core/src/main/java/org/apache/doris/planner/external/HudiScanProvider.java
index 59274ec521..355ea6c732 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/planner/external/HudiScanProvider.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/planner/external/HudiScanProvider.java
@@ -17,13 +17,19 @@
package org.apache.doris.planner.external;
+import org.apache.doris.analysis.Expr;
import org.apache.doris.analysis.TupleDescriptor;
import org.apache.doris.catalog.external.HMSExternalTable;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.MetaNotFoundException;
+import org.apache.doris.common.UserException;
import org.apache.doris.planner.ColumnRange;
import org.apache.doris.thrift.TFileFormatType;
+import com.google.common.collect.Lists;
+import org.apache.hadoop.mapred.FileSplit;
+import org.apache.hadoop.mapred.InputSplit;
+
import java.util.Collections;
import java.util.List;
import java.util.Map;
@@ -48,4 +54,19 @@ public class HudiScanProvider extends HiveScanProvider {
public List<String> getPathPartitionKeys() throws DdlException,
MetaNotFoundException {
return Collections.emptyList();
}
+
+ @Override
+ public List<InputSplit> getSplits(List<Expr> exprs) throws UserException {
+ List<InputSplit> splits = super.getSplits(exprs);
+ List<InputSplit> filterSplits = Lists.newArrayList();
+ for (InputSplit split : splits) {
+ FileSplit fileSplit = (FileSplit) split;
+ // skip hidden files start with "."
+ if (fileSplit.getPath().getName().startsWith(".")) {
+ continue;
+ }
+ filterSplits.add(split);
+ }
+ return filterSplits;
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
