This is an automated email from the ASF dual-hosted git repository.
liugddx pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/seatunnel.git
The following commit(s) were added to refs/heads/dev by this push:
new de7b86a5dd [fix][hive-source][bug] fix An error occurred reading an
empty directory (#5427)
de7b86a5dd is described below
commit de7b86a5dd24d9cff4fa28ca7b8eadbd50ed8bb9
Author: ZhilinLi <[email protected]>
AuthorDate: Tue Sep 12 15:21:05 2023 +0800
[fix][hive-source][bug] fix An error occurred reading an empty directory
(#5427)
* [fix][hive-source][bug] fix An error occurred reading an empty directory
* [fix][hive-source][bug] fix An error occurred reading an empty directory
---
.../file/source/reader/AbstractReadStrategy.java | 22 ++++++++--------------
.../file/source/reader/ExcelReadStrategy.java | 3 ++-
.../file/source/reader/TextReadStrategy.java | 3 ++-
3 files changed, 12 insertions(+), 16 deletions(-)
diff --git
a/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/AbstractReadStrategy.java
b/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/AbstractReadStrategy.java
index e4e1694f30..1dc8e7a2ad 100644
---
a/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/AbstractReadStrategy.java
+++
b/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/AbstractReadStrategy.java
@@ -24,8 +24,6 @@ import org.apache.seatunnel.api.table.type.SeaTunnelDataType;
import org.apache.seatunnel.api.table.type.SeaTunnelRowType;
import org.apache.seatunnel.connectors.seatunnel.file.config.BaseSourceConfig;
import org.apache.seatunnel.connectors.seatunnel.file.config.HadoopConf;
-import
org.apache.seatunnel.connectors.seatunnel.file.exception.FileConnectorErrorCode;
-import
org.apache.seatunnel.connectors.seatunnel.file.exception.FileConnectorException;
import
org.apache.seatunnel.connectors.seatunnel.file.sink.util.FileSystemUtils;
import org.apache.hadoop.conf.Configuration;
@@ -153,15 +151,9 @@ public abstract class AbstractReadStrategy implements
ReadStrategy {
}
}
}
-
- if (fileNames.isEmpty()) {
- throw new FileConnectorException(
- FileConnectorErrorCode.FILE_LIST_EMPTY,
- "The target file list is empty,"
- + "SeaTunnel will not be able to sync empty table,
"
- + "please check the configuration parameters such
as: [file_filter_pattern]");
+ if (this.fileNames.isEmpty()) {
+ log.error("The current directory is empty " + path);
}
-
return fileNames;
}
@@ -196,10 +188,12 @@ public abstract class AbstractReadStrategy implements
ReadStrategy {
protected Map<String, String> parsePartitionsByPath(String path) {
LinkedHashMap<String, String> partitions = new LinkedHashMap<>();
- Arrays.stream(path.split("/", -1))
- .filter(split -> split.contains("="))
- .map(split -> split.split("=", -1))
- .forEach(kv -> partitions.put(kv[0], kv[1]));
+ if (path != null && !path.isEmpty()) {
+ Arrays.stream(path.split("/", -1))
+ .filter(split -> split.contains("="))
+ .map(split -> split.split("=", -1))
+ .forEach(kv -> partitions.put(kv[0], kv[1]));
+ }
return partitions;
}
diff --git
a/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/ExcelReadStrategy.java
b/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/ExcelReadStrategy.java
index 0b1cfc083b..649216c752 100644
---
a/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/ExcelReadStrategy.java
+++
b/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/ExcelReadStrategy.java
@@ -136,7 +136,8 @@ public class ExcelReadStrategy extends AbstractReadStrategy
{
"Schmea information is not set or incorrect schmea
settings");
}
SeaTunnelRowType userDefinedRowTypeWithPartition =
- mergePartitionTypes(fileNames.get(0), seaTunnelRowType);
+ mergePartitionTypes(
+ fileNames.size() > 0 ? fileNames.get(0) : null,
seaTunnelRowType);
// column projection
if (pluginConfig.hasPath(BaseSourceConfig.READ_COLUMNS.key())) {
// get the read column index from user-defined row type
diff --git
a/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/TextReadStrategy.java
b/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/TextReadStrategy.java
index 4b931cb890..eefa8e4de7 100644
---
a/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/TextReadStrategy.java
+++
b/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/TextReadStrategy.java
@@ -138,7 +138,8 @@ public class TextReadStrategy extends AbstractReadStrategy {
@Override
public void setSeaTunnelRowTypeInfo(SeaTunnelRowType seaTunnelRowType) {
SeaTunnelRowType userDefinedRowTypeWithPartition =
- mergePartitionTypes(fileNames.get(0), seaTunnelRowType);
+ mergePartitionTypes(
+ fileNames.size() > 0 ? fileNames.get(0) : null,
seaTunnelRowType);
if (pluginConfig.hasPath(BaseSourceConfig.DELIMITER.key())) {
fieldDelimiter =
pluginConfig.getString(BaseSourceConfig.DELIMITER.key());
} else {
