This is an automated email from the ASF dual-hosted git repository.
rong pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 808f5752a9d Load: fix some minor issues (#13781)
808f5752a9d is described below
commit 808f5752a9df83e623a4bcaddfb89c1f9c5c352f
Author: Zikun Ma <[email protected]>
AuthorDate: Wed Oct 16 10:32:00 2024 +0800
Load: fix some minor issues (#13781)
---
.../plan/analyze/load/LoadTsFileAnalyzer.java | 6 ++-
.../load/LoadTsFileToTableModelAnalyzer.java | 10 ++++-
.../load/LoadTsFileToTreeModelAnalyzer.java | 2 +
.../plan/relational/sql/parser/AstBuilder.java | 44 ++++++++--------------
.../plan/relational/sql/util/SqlFormatter.java | 32 ++++++++--------
.../db/relational/grammar/sql/RelationalSql.g4 | 6 +--
6 files changed, 49 insertions(+), 51 deletions(-)
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileAnalyzer.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileAnalyzer.java
index 6e965c80b01..9a65349ab34 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileAnalyzer.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileAnalyzer.java
@@ -20,6 +20,7 @@
package org.apache.iotdb.db.queryengine.plan.analyze.load;
import org.apache.iotdb.commons.auth.AuthException;
+import org.apache.iotdb.db.exception.VerifyMetadataException;
import org.apache.iotdb.db.exception.sql.SemanticException;
import org.apache.iotdb.db.queryengine.common.MPPQueryContext;
import org.apache.iotdb.db.queryengine.plan.analyze.ClusterPartitionFetcher;
@@ -143,7 +144,7 @@ public abstract class LoadTsFileAnalyzer implements
AutoCloseable {
} catch (Exception e) {
final String exceptionMessage =
String.format(
- "The file %s is not a valid tsfile. Please check the input
file. Detail: %s",
+ "Loading file %s failed. Detail: %s",
tsFile.getPath(), e.getMessage() == null ?
e.getClass().getName() : e.getMessage());
LOGGER.warn(exceptionMessage, e);
analysis.setFinishQueryAfterAnalyze(true);
@@ -154,7 +155,8 @@ public abstract class LoadTsFileAnalyzer implements
AutoCloseable {
return true;
}
- protected abstract void analyzeSingleTsFile(final File tsFile) throws
IOException, AuthException;
+ protected abstract void analyzeSingleTsFile(final File tsFile)
+ throws IOException, AuthException, VerifyMetadataException;
protected TsFileResource constructTsFileResource(
final TsFileSequenceReader reader, final File tsFile) throws IOException
{
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileToTableModelAnalyzer.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileToTableModelAnalyzer.java
index 7316e7a3de1..27c7efa1b23 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileToTableModelAnalyzer.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileToTableModelAnalyzer.java
@@ -115,7 +115,8 @@ public class LoadTsFileToTableModelAnalyzer extends
LoadTsFileAnalyzer {
}
@Override
- protected void analyzeSingleTsFile(final File tsFile) throws IOException,
AuthException {
+ protected void analyzeSingleTsFile(final File tsFile)
+ throws IOException, AuthException, VerifyMetadataException {
try (final TsFileSequenceReader reader = new
TsFileSequenceReader(tsFile.getAbsolutePath())) {
// can be reused when constructing tsfile resource
final TsFileSequenceReaderTimeseriesMetadataIterator
timeseriesMetadataIterator =
@@ -127,6 +128,8 @@ public class LoadTsFileToTableModelAnalyzer extends
LoadTsFileAnalyzer {
}
// check whether the tsfile is table-model or not
+ // TODO: currently, loading a file with both tree-model and table-model
data is not supported.
+ // May need to support this and remove this check in the future.
if (Objects.isNull(reader.readFileMetadata().getTableSchemaMap())
|| reader.readFileMetadata().getTableSchemaMap().size() == 0) {
throw new SemanticException("Attempted to load a tree-model TsFile
into table-model.");
@@ -147,7 +150,10 @@ public class LoadTsFileToTableModelAnalyzer extends
LoadTsFileAnalyzer {
true)
.orElse(null);
if (Objects.isNull(realSchema)) {
- LOGGER.warn("Failed to validata schema for table {}",
name2Schema.getValue());
+ throw new VerifyMetadataException(
+ String.format(
+ "Failed to validate schema for table {%s, %s}",
+ name2Schema.getKey(), name2Schema.getValue()));
}
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileToTreeModelAnalyzer.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileToTreeModelAnalyzer.java
index 5552109999d..6e2382dc30d 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileToTreeModelAnalyzer.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileToTreeModelAnalyzer.java
@@ -119,6 +119,8 @@ public class LoadTsFileToTreeModelAnalyzer extends
LoadTsFileAnalyzer {
}
// check whether the tsfile is tree-model or not
+ // TODO: currently, loading a file with both tree-model and table-model
data is not supported.
+ // May need to support this and remove this check in the future.
if (Objects.nonNull(reader.readFileMetadata().getTableSchemaMap())
&& reader.readFileMetadata().getTableSchemaMap().size() != 0) {
throw new SemanticException("Attempted to load a table-model TsFile
into tree-model.");
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java
index 72bf42f7ed5..800fe7ea7ca 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java
@@ -561,40 +561,26 @@ public class AstBuilder extends
RelationalSqlBaseVisitor<Node> {
@Override
public Node
visitLoadTsFileStatement(RelationalSqlParser.LoadTsFileStatementContext ctx) {
- final Map<String, String> loadTsFileAttributes = new HashMap<>();
- if (ctx.loadFileWithAttributeClauses() != null) {
- for (RelationalSqlParser.LoadFileWithAttributeClauseContext
attributeContext :
- ctx.loadFileWithAttributeClauses().loadFileWithAttributeClause()) {
- final String key =
-
parseStringLiteral(attributeContext.loadFileWithKey.getText()).trim().toLowerCase();
- final String value =
-
parseStringLiteral(attributeContext.loadFileWithValue.getText()).trim().toLowerCase();
-
- LoadTsFileConfigurator.validateParameters(key, value);
- loadTsFileAttributes.put(key, value);
- }
- }
+ final Map<String, String> withAttributes =
+ ctx.loadFileWithAttributesClause() != null
+ ? parseLoadFileWithAttributeClauses(
+
ctx.loadFileWithAttributesClause().loadFileWithAttributeClause())
+ : new HashMap<>();
+ withAttributes.forEach(LoadTsFileConfigurator::validateParameters);
return new LoadTsFile(
- getLocation(ctx), ((StringLiteral) visit(ctx.fileName)).getValue(),
loadTsFileAttributes);
+ getLocation(ctx), ((StringLiteral) visit(ctx.fileName)).getValue(),
withAttributes);
}
- public static String parseStringLiteral(String src) {
- if (2 <= src.length()) {
- // do not unescape string
- String unWrappedString = src.substring(1, src.length() - 1);
- if (src.charAt(0) == '\"' && src.charAt(src.length() - 1) == '\"') {
- // replace "" with "
- String replaced = unWrappedString.replace("\"\"", "\"");
- return replaced.length() == 0 ? "" : replaced;
- }
- if ((src.charAt(0) == '\'' && src.charAt(src.length() - 1) == '\'')) {
- // replace '' with '
- String replaced = unWrappedString.replace("''", "'");
- return replaced.length() == 0 ? "" : replaced;
- }
+ private Map<String, String> parseLoadFileWithAttributeClauses(
+ List<RelationalSqlParser.LoadFileWithAttributeClauseContext> contexts) {
+ final Map<String, String> withAttributesMap = new HashMap<>();
+ for (RelationalSqlParser.LoadFileWithAttributeClauseContext context :
contexts) {
+ withAttributesMap.put(
+ ((StringLiteral) visit(context.loadFileWithKey)).getValue(),
+ ((StringLiteral) visit(context.loadFileWithValue)).getValue());
}
- return src;
+ return withAttributesMap;
}
@Override
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/util/SqlFormatter.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/util/SqlFormatter.java
index 355f56cdea1..e5b0fb3e969 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/util/SqlFormatter.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/util/SqlFormatter.java
@@ -804,21 +804,23 @@ public final class SqlFormatter {
builder.append("\"" + node.getFilePath() + "\"");
builder.append(" \n");
- builder
- .append("WITH (")
- .append("\n")
- .append(
- node.getLoadAttributes().entrySet().stream()
- .map(
- entry ->
- indentString(1)
- + "\""
- + entry.getKey()
- + "\" = \""
- + entry.getValue()
- + "\"")
- .collect(joining(", " + "\n")))
- .append(")\n");
+ if (!node.getLoadAttributes().isEmpty()) {
+ builder
+ .append("WITH (")
+ .append("\n")
+ .append(
+ node.getLoadAttributes().entrySet().stream()
+ .map(
+ entry ->
+ indentString(1)
+ + "\""
+ + entry.getKey()
+ + "\" = \""
+ + entry.getValue()
+ + "\"")
+ .collect(joining(", " + "\n")))
+ .append(")\n");
+ }
return null;
}
diff --git
a/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4
b/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4
index 8984ccaea77..a3cd45ad653 100644
---
a/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4
+++
b/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4
@@ -240,10 +240,10 @@ showFunctionsStatement
// -------------------------------------------- Load Statement
---------------------------------------------------------
loadTsFileStatement
- : LOAD fileName=string (loadFileWithAttributeClauses)?
+ : LOAD fileName=string (loadFileWithAttributesClause)?
;
-loadFileWithAttributeClauses
+loadFileWithAttributesClause
: WITH
'('
(loadFileWithAttributeClause ',')* loadFileWithAttributeClause?
@@ -251,7 +251,7 @@ loadFileWithAttributeClauses
;
loadFileWithAttributeClause
- : loadFileWithKey=STRING EQ loadFileWithValue=STRING
+ : loadFileWithKey=string EQ loadFileWithValue=string
;
