imay commented on a change in pull request #1592: Broker load supports function
URL: https://github.com/apache/incubator-doris/pull/1592#discussion_r311885591
##########
File path: fe/src/main/java/org/apache/doris/analysis/DataDescription.java
##########
@@ -189,92 +186,100 @@ public boolean isPullLoad() {
return isPullLoad;
}
- private void checkColumnInfo() throws AnalysisException {
- if (columnNames == null || columnNames.isEmpty()) {
+ /**
+ * Analyze parsedExprMap and columnToFunction from columns and
columnMappingList
+ * Example: columns (col1, tmp_col2, tmp_col3) set (col2=tmp_col2+1,
col3=strftime("%Y-%m-%d %H:%M:%S", tmp_col3))
+ * Result: parsedExprMap = {"col1": null, "tmp_col2": null, "tmp_col3":
null,
+ * "col2": "tmp_col2+1", "col3": "strftime("%Y-%m-%d %H:%M:%S", tmp_col3)"}
+ */
+ private void analyzeColumns() throws AnalysisException {
+ if (columns == null || columns.isEmpty()) {
return;
}
- Set<String> columnSet = Sets.newTreeSet(String.CASE_INSENSITIVE_ORDER);
- for (String col : columnNames) {
- if (!columnSet.add(col)) {
-
ErrorReport.reportAnalysisException(ErrorCode.ERR_DUP_FIELDNAME, col);
+ // merge columns exprs from columns and columnMappingList
+ // used to check duplicated column name
+ Set<String> columnNames = Sets.newHashSet();
+ parsedColumnExprList = Lists.newArrayList();
+ // Step1: analyze columns
+ for (String columnName : columns) {
+ if (!columnNames.add(columnName)) {
+ throw new AnalysisException("Duplicate column : " +
columnName);
}
+ ImportColumnDesc importColumnDesc = new
ImportColumnDesc(columnName, null);
+ parsedColumnExprList.add(importColumnDesc);
}
- }
- private void checkColumnMapping() throws AnalysisException {
+
if (columnMappingList == null || columnMappingList.isEmpty()) {
return;
}
-
+ // Step2: analyze column mapping
+ // the column expr only support the SlotRef or eq binary predicate
which's child(0) must be a SloRef.
+ // the duplicate column name of SloRef is forbidden.
columnToFunction = Maps.newTreeMap(String.CASE_INSENSITIVE_ORDER);
- parsedExprMap = Maps.newHashMap();
- for (Expr expr : columnMappingList) {
- if (!(expr instanceof BinaryPredicate)) {
- throw new AnalysisException("Mapping function expr error.
expr: " + expr.toSql());
- }
+ for (Expr columnExpr : columnMappingList) {
- BinaryPredicate predicate = (BinaryPredicate) expr;
+ if (!(columnExpr instanceof BinaryPredicate)) {
+ throw new AnalysisException("Mapping function expr only
support the column or eq binary predicate. "
+ + "Expr: " +
columnExpr.toSql());
+ }
+ BinaryPredicate predicate = (BinaryPredicate) columnExpr;
if (predicate.getOp() != Operator.EQ) {
- throw new AnalysisException("Mapping function operator error.
op: " + predicate.getOp());
+ throw new AnalysisException("Mapping function expr only
support the column or eq binary predicate. "
+ + "The mapping operator
error, op: " + predicate.getOp());
}
-
Expr child0 = predicate.getChild(0);
if (!(child0 instanceof SlotRef)) {
- throw new AnalysisException("Mapping column error. column: " +
child0.toSql());
+ throw new AnalysisException("Mapping function expr only
support the column or eq binary predicate. "
+ + "The mapping column
error. column: " + child0.toSql());
}
-
String column = ((SlotRef) child0).getColumnName();
- if (columnToFunction.containsKey(column)) {
+ if (!columnNames.add(column)) {
throw new AnalysisException("Duplicate column mapping: " +
column);
}
-
- // we support function and column reference to change a column name
+ // hadoop load only supports the FunctionCallExpr
Expr child1 = predicate.getChild(1);
- if (!(child1 instanceof FunctionCallExpr)) {
- if (isPullLoad && child1 instanceof SlotRef) {
- // we only support SlotRef in pull load
- } else {
- throw new AnalysisException("Mapping function error,
function: " + child1.toSql());
- }
- }
-
- if (!child1.supportSerializable()) {
- throw new AnalysisException("Expr do not support
serializable." + child1.toSql());
+ if (!isPullLoad && !(child1 instanceof FunctionCallExpr)) {
+ throw new AnalysisException("Hadoop load only supports the
designated function. "
+ + "The error mapping
function is:" + child1.toSql());
}
+ ImportColumnDesc importColumnDesc = new ImportColumnDesc(column,
child1);
+ parsedColumnExprList.add(importColumnDesc);
+ analyzeColumnToFunction(column, child1);
Review comment:
If this is not a hadoop load, do we need to call this function ?
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
