This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-1.2-lts by this push:
new 87840fa902 [fix](multicatalog) fix protocol of reading data from cosn
for hive/iceberg catalog (#21683)
87840fa902 is described below
commit 87840fa902e2ee173371d457dc0d085724fc693d
Author: Yulei-Yang <[email protected]>
AuthorDate: Mon Jul 10 22:10:45 2023 +0800
[fix](multicatalog) fix protocol of reading data from cosn for hive/iceberg
catalog (#21683)
Error msg:
ERROR 1105 (HY000): errCode = 2, detailMessage = get file split failed for
table: xxxx, err: org.apache.hadoop.fs.s3a.RemoteFileChangedException: open
`s3a://xxx-bucket/xxx.db/xxx_table/dt=20230704/000000_0': Change reported by S3
during open at position 198924776. ETag e42667ea1cbdfcfacb46fa1627356bce was
unavailable
Reason: tencent cloud cosn(CHDFS) is not applicable accessed by s3
protocol, it's more like a hdfs system.
---
.../java/org/apache/doris/common/FeConstants.java | 1 +
.../doris/datasource/hive/HiveMetaStoreCache.java | 12 ++++++------
.../doris/planner/external/HiveScanProvider.java | 19 ++++++++++---------
.../planner/external/iceberg/IcebergScanProvider.java | 19 ++++++++++---------
4 files changed, 27 insertions(+), 24 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/FeConstants.java
b/fe/fe-core/src/main/java/org/apache/doris/common/FeConstants.java
index 8d032b604a..91ad6e4a61 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/FeConstants.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/FeConstants.java
@@ -75,6 +75,7 @@ public class FeConstants {
public static String FS_PREFIX_OSS = "oss";
public static String FS_PREFIX_BOS = "bos";
public static String FS_PREFIX_COS = "cos";
+ public static String FS_PREFIX_COSN = "cosn";
public static String FS_PREFIX_OBS = "obs";
public static String FS_PREFIX_OFS = "ofs";
public static String FS_PREFIX_GFS = "gfs";
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java
index 8b7d7f83c6..778fb10262 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java
@@ -311,12 +311,12 @@ public class HiveMetaStoreCache {
// convert oss:// to s3://
private String convertToS3IfNecessary(String location) {
LOG.debug("try convert location to s3 prefix: " + location);
- if (location.startsWith(FeConstants.FS_PREFIX_COS)
- || location.startsWith(FeConstants.FS_PREFIX_BOS)
- || location.startsWith(FeConstants.FS_PREFIX_BOS)
- || location.startsWith(FeConstants.FS_PREFIX_OSS)
- || location.startsWith(FeConstants.FS_PREFIX_S3A)
- || location.startsWith(FeConstants.FS_PREFIX_S3N)) {
+ if ((location.startsWith(FeConstants.FS_PREFIX_COS)) &&
!(location.startsWith(FeConstants.FS_PREFIX_COSN))
+ || location.startsWith(FeConstants.FS_PREFIX_BOS)
+ || location.startsWith(FeConstants.FS_PREFIX_BOS)
+ || location.startsWith(FeConstants.FS_PREFIX_OSS)
+ || location.startsWith(FeConstants.FS_PREFIX_S3A)
+ || location.startsWith(FeConstants.FS_PREFIX_S3N)) {
int pos = location.indexOf("://");
if (pos == -1) {
throw new RuntimeException("No '://' found in location: " +
location);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/planner/external/HiveScanProvider.java
b/fe/fe-core/src/main/java/org/apache/doris/planner/external/HiveScanProvider.java
index 9aa4ab6dc4..845d62f92c 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/planner/external/HiveScanProvider.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/planner/external/HiveScanProvider.java
@@ -112,16 +112,17 @@ public class HiveScanProvider extends
HMSTableScanProvider {
public TFileType getLocationType() throws DdlException,
MetaNotFoundException {
String location = hmsTable.getRemoteTable().getSd().getLocation();
if (location != null && !location.isEmpty()) {
- if (location.startsWith(FeConstants.FS_PREFIX_S3)
- || location.startsWith(FeConstants.FS_PREFIX_S3A)
- || location.startsWith(FeConstants.FS_PREFIX_S3N)
- || location.startsWith(FeConstants.FS_PREFIX_BOS)
- || location.startsWith(FeConstants.FS_PREFIX_COS)
- || location.startsWith(FeConstants.FS_PREFIX_OSS)
- || location.startsWith(FeConstants.FS_PREFIX_OBS)) {
- return TFileType.FILE_S3;
- } else if (location.startsWith(FeConstants.FS_PREFIX_HDFS)) {
+ if (location.startsWith(FeConstants.FS_PREFIX_HDFS)
+ || location.startsWith(FeConstants.FS_PREFIX_COSN)) {
return TFileType.FILE_HDFS;
+ } else if (location.startsWith(FeConstants.FS_PREFIX_S3)
+ || location.startsWith(FeConstants.FS_PREFIX_S3A)
+ || location.startsWith(FeConstants.FS_PREFIX_S3N)
+ || location.startsWith(FeConstants.FS_PREFIX_BOS)
+ || location.startsWith(FeConstants.FS_PREFIX_COS)
+ || location.startsWith(FeConstants.FS_PREFIX_OSS)
+ || location.startsWith(FeConstants.FS_PREFIX_OBS)) {
+ return TFileType.FILE_S3;
} else if (location.startsWith(FeConstants.FS_PREFIX_FILE)) {
return TFileType.FILE_LOCAL;
} else if (location.startsWith(FeConstants.FS_PREFIX_OFS)) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/planner/external/iceberg/IcebergScanProvider.java
b/fe/fe-core/src/main/java/org/apache/doris/planner/external/iceberg/IcebergScanProvider.java
index 970dc97800..e8295a3fc3 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/planner/external/iceberg/IcebergScanProvider.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/planner/external/iceberg/IcebergScanProvider.java
@@ -117,16 +117,17 @@ public class IcebergScanProvider extends
QueryScanProvider {
public TFileType getLocationType() throws DdlException,
MetaNotFoundException {
String location = icebergSource.getIcebergTable().location();
if (location != null && !location.isEmpty()) {
- if (location.startsWith(FeConstants.FS_PREFIX_S3)
- || location.startsWith(FeConstants.FS_PREFIX_S3A)
- || location.startsWith(FeConstants.FS_PREFIX_S3N)
- || location.startsWith(FeConstants.FS_PREFIX_BOS)
- || location.startsWith(FeConstants.FS_PREFIX_COS)
- || location.startsWith(FeConstants.FS_PREFIX_OSS)
- || location.startsWith(FeConstants.FS_PREFIX_OBS)) {
- return TFileType.FILE_S3;
- } else if (location.startsWith(FeConstants.FS_PREFIX_HDFS)) {
+ if (location.startsWith(FeConstants.FS_PREFIX_HDFS)
+ || location.startsWith(FeConstants.FS_PREFIX_COSN)) {
return TFileType.FILE_HDFS;
+ } else if (location.startsWith(FeConstants.FS_PREFIX_S3)
+ || location.startsWith(FeConstants.FS_PREFIX_S3A)
+ || location.startsWith(FeConstants.FS_PREFIX_S3N)
+ || location.startsWith(FeConstants.FS_PREFIX_BOS)
+ || location.startsWith(FeConstants.FS_PREFIX_COS)
+ || location.startsWith(FeConstants.FS_PREFIX_OSS)
+ || location.startsWith(FeConstants.FS_PREFIX_OBS)) {
+ return TFileType.FILE_S3;
} else if (location.startsWith(FeConstants.FS_PREFIX_FILE)) {
return TFileType.FILE_LOCAL;
} else if (location.startsWith(FeConstants.FS_PREFIX_OFS)) {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
