This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new bf42b594fdb branch-3.0: [fix](hive) ignore all hidden dir and files
#52286 (#52323)
bf42b594fdb is described below
commit bf42b594fdbca98ca89841e33508b4e98bcfbd18
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Thu Jun 26 14:11:32 2025 +0800
branch-3.0: [fix](hive) ignore all hidden dir and files #52286 (#52323)
Cherry-picked from #52286
Co-authored-by: Mingyu Chen (Rayner) <[email protected]>
---
.../doris/datasource/hive/HiveMetaStoreCache.java | 18 ++++--------------
.../org/apache/doris/datasource/PathVisibleTest.java | 7 +++++--
2 files changed, 9 insertions(+), 16 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java
index de867bfc2b9..314162d3f0d 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java
@@ -1056,34 +1056,24 @@ public class HiveMetaStoreCache {
return false;
}
String pathStr = path.toUri().toString();
- if (containsHiddenPath(pathStr) || path.getName().startsWith("_"))
{
+ if (containsHiddenPath(pathStr)) {
return false;
}
- for (String name : pathStr.split("/")) {
- if (isGeneratedPath(name)) {
- return false;
- }
- }
return true;
}
private static boolean containsHiddenPath(String path) {
- if (path.startsWith(".")) {
+ // Hive ignores files starting with _ and .
+ if (path.startsWith(".") || path.startsWith("_")) {
return true;
}
for (int i = 0; i < path.length() - 1; i++) {
- if (path.charAt(i) == '/' && path.charAt(i + 1) == '.') {
+ if (path.charAt(i) == '/' && (path.charAt(i + 1) == '.' ||
path.charAt(i + 1) == '_')) {
return true;
}
}
return false;
}
-
- private static boolean isGeneratedPath(String name) {
- return "_temporary".equals(name) // generated by spark
- || "_imapala_insert_staging".equals(name) // generated by
impala
- || name.startsWith("."); // generated by hive or hidden
file
- }
}
@Data
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/datasource/PathVisibleTest.java
b/fe/fe-core/src/test/java/org/apache/doris/datasource/PathVisibleTest.java
index 0937bbc3cc8..6610a5f2521 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/datasource/PathVisibleTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/PathVisibleTest.java
@@ -31,17 +31,20 @@ public class PathVisibleTest {
Assert.assertFalse(FileCacheValue.isFileVisible(new
Path("/visible/.hidden/path")));
Assert.assertFalse(FileCacheValue.isFileVisible(new
Path("hdfs://visible/path/.file")));
Assert.assertFalse(FileCacheValue.isFileVisible(new
Path("/visible/path/_temporary_xx")));
- Assert.assertFalse(FileCacheValue.isFileVisible(new
Path("/visible/path/_imapala_insert_staging")));
+ Assert.assertFalse(FileCacheValue.isFileVisible(new
Path("/visible/path/_impala_insert_staging")));
Assert.assertFalse(FileCacheValue.isFileVisible(new
Path("/visible//.hidden/path")));
Assert.assertFalse(FileCacheValue.isFileVisible(new
Path("s3://visible/.hidden/path")));
Assert.assertFalse(FileCacheValue.isFileVisible(new
Path("///visible/path/.file")));
Assert.assertFalse(FileCacheValue.isFileVisible(new
Path("/visible/path///_temporary_xx")));
- Assert.assertFalse(FileCacheValue.isFileVisible(new
Path("hdfs://visible//path/_imapala_insert_staging")));
+ Assert.assertFalse(FileCacheValue.isFileVisible(new
Path("hdfs://visible//path/_impala_insert_staging")));
+ Assert.assertFalse(FileCacheValue.isFileVisible(
+ new
Path("hdfs://hacluster/user/hive/warehouse/db1.db/tbl1/_spark_metadata/")));
Assert.assertTrue(FileCacheValue.isFileVisible(new
Path("s3://visible/path")));
Assert.assertTrue(FileCacheValue.isFileVisible(new Path("path")));
Assert.assertTrue(FileCacheValue.isFileVisible(new
Path("hdfs://visible/path./1.txt")));
Assert.assertTrue(FileCacheValue.isFileVisible(new Path("/1.txt")));
+ Assert.assertTrue(FileCacheValue.isFileVisible(new
Path("hdfs://vis_ible_/pa.th./1_.txt__")));
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
