This is an automated email from the ASF dual-hosted git repository. dlych pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit a81d38112c3c8c114432c5039c0b752a1954dbdd Author: Ali Alsuliman <[email protected]> AuthorDate: Tue Apr 5 23:55:19 2022 +0300 [NO ISSUE][OTH] Redact field name - user model changes: no - storage format changes: no - interface changes: no Details: Change-Id: I360681ff4e1460561eb5b614ce16f7d6519096b2 Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/16023 Tested-by: Jenkins <[email protected]> Integration-Tests: Jenkins <[email protected]> Reviewed-by: Ali Alsuliman <[email protected]> Reviewed-by: Michael Blow <[email protected]> --- .../optimizer/rules/ConstantFoldingRule.java | 3 +- .../optimizer/rules/RemoveDuplicateFieldsRule.java | 3 +- .../SqlppExpressionToPlanTranslator.java | 4 +- .../asterix/translator/util/ValidateUtil.java | 50 +++++++++++++++------- .../asterix/app/translator/QueryTranslator.java | 31 +++++++++----- .../src/test/resources/metadata/testsuite.xml | 12 +++--- .../test/resources/runtimets/testsuite_sqlpp.xml | 6 +-- .../external/library/java/base/JRecord.java | 4 +- .../asterix/external/parser/ADMDataParser.java | 5 ++- .../external/parser/AbstractJsonDataParser.java | 6 ++- .../external/parser/AbstractNestedDataParser.java | 3 +- .../external/parser/DelimitedDataParser.java | 4 +- .../asterix/external/parser/TweetParser.java | 3 +- .../utils/RTreeResourceFactoryProvider.java | 5 ++- .../apache/asterix/metadata/utils/TypeUtil.java | 10 +++-- .../impl/ClosedRecordConstructorResultType.java | 4 +- .../impl/OpenRecordConstructorResultType.java | 4 +- .../typecomputer/impl/RecordMergeTypeComputer.java | 6 ++- .../om/types/runtime/RuntimeRecordTypeInfo.java | 4 +- 19 files changed, 109 insertions(+), 58 deletions(-) diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ConstantFoldingRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ConstantFoldingRule.java index 0c2eeba963..bdee0ed38b 100644 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ConstantFoldingRule.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ConstantFoldingRule.java @@ -98,6 +98,7 @@ import org.apache.hyracks.api.exceptions.Warning; import org.apache.hyracks.data.std.api.IPointable; import org.apache.hyracks.data.std.primitive.VoidPointable; import org.apache.hyracks.dataflow.common.comm.util.ByteBufferInputStream; +import org.apache.hyracks.util.LogRedactionUtil; import com.google.common.collect.ImmutableMap; @@ -334,7 +335,7 @@ public class ConstantFoldingRule implements IAlgebraicRewriteRule { IWarningCollector warningCollector = optContext.getWarningCollector(); if (warningCollector.shouldWarn()) { warningCollector.warn(Warning.of(fieldNameExpr.second.getSourceLocation(), - ErrorCode.COMPILATION_DUPLICATE_FIELD_NAME, fieldName)); + ErrorCode.COMPILATION_DUPLICATE_FIELD_NAME, LogRedactionUtil.userData(fieldName))); } iterator.remove(); iterator.next(); diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/RemoveDuplicateFieldsRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/RemoveDuplicateFieldsRule.java index 6c4f8c6a59..e924a4cef8 100644 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/RemoveDuplicateFieldsRule.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/RemoveDuplicateFieldsRule.java @@ -41,6 +41,7 @@ import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractLogi import org.apache.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule; import org.apache.hyracks.api.exceptions.IWarningCollector; import org.apache.hyracks.api.exceptions.Warning; +import org.apache.hyracks.util.LogRedactionUtil; /** * <pre> @@ -116,7 +117,7 @@ public class RemoveDuplicateFieldsRule implements IAlgebraicRewriteRule { IWarningCollector warningCollector = context.getWarningCollector(); if (warningCollector.shouldWarn()) { warningCollector.warn(Warning.of(fieldNameExpr.getSourceLocation(), - ErrorCode.COMPILATION_DUPLICATE_FIELD_NAME, fieldName)); + ErrorCode.COMPILATION_DUPLICATE_FIELD_NAME, LogRedactionUtil.userData(fieldName))); } iterator.remove(); iterator.next(); diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/SqlppExpressionToPlanTranslator.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/SqlppExpressionToPlanTranslator.java index 95026a5539..516780de23 100644 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/SqlppExpressionToPlanTranslator.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/SqlppExpressionToPlanTranslator.java @@ -136,6 +136,7 @@ import org.apache.hyracks.algebricks.core.algebra.operators.logical.WindowOperat import org.apache.hyracks.algebricks.core.algebra.plan.ALogicalPlanImpl; import org.apache.hyracks.algebricks.core.algebra.util.OperatorManipulationUtil; import org.apache.hyracks.api.exceptions.SourceLocation; +import org.apache.hyracks.util.LogRedactionUtil; /** * Each visit returns a pair of an operator and a variable. The variable @@ -939,7 +940,8 @@ public class SqlppExpressionToPlanTranslator extends LangExpressionToPlanTransla private FieldBinding generateFieldBinding(String fieldName, Expression fieldValueExpr, Set<String> outFieldNames, SourceLocation sourceLoc) throws CompilationException { if (!outFieldNames.add(fieldName)) { - throw new CompilationException(ErrorCode.DUPLICATE_FIELD_NAME, sourceLoc, fieldName); + throw new CompilationException(ErrorCode.DUPLICATE_FIELD_NAME, sourceLoc, + LogRedactionUtil.userData(fieldName)); } return new FieldBinding(new LiteralExpr(new StringLiteral(fieldName)), fieldValueExpr); } diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/util/ValidateUtil.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/util/ValidateUtil.java index b32b05e1c8..b33026f5b2 100644 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/util/ValidateUtil.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/util/ValidateUtil.java @@ -32,6 +32,7 @@ import org.apache.asterix.om.types.IAType; import org.apache.asterix.om.utils.RecordUtil; import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException; import org.apache.hyracks.api.exceptions.SourceLocation; +import org.apache.hyracks.util.LogRedactionUtil; /** * A util that can verify if a filter field, a list of partitioning expressions, @@ -66,7 +67,7 @@ public class ValidateUtil { IAType fieldType = itemType.getSubFieldType(filterField); if (fieldType == null) { throw new CompilationException(ErrorCode.COMPILATION_FIELD_NOT_FOUND, sourceLoc, - RecordUtil.toFullyQualifiedName(filterField)); + LogRedactionUtil.userData(RecordUtil.toFullyQualifiedName(filterField))); } switch (fieldType.getTypeTag()) { case TINYINT: @@ -86,7 +87,7 @@ public class ValidateUtil { break; case UNION: throw new CompilationException(ErrorCode.COMPILATION_FILTER_CANNOT_BE_NULLABLE, - RecordUtil.toFullyQualifiedName(filterField)); + LogRedactionUtil.userData(RecordUtil.toFullyQualifiedName(filterField))); default: throw new CompilationException(ErrorCode.COMPILATION_ILLEGAL_FILTER_TYPE, fieldType.getTypeTag().name()); @@ -128,7 +129,8 @@ public class ValidateUtil { IAType fieldType = recType.getSubFieldType(fieldName); if (fieldType == null) { String unTypeField = fieldName.get(0) == null ? "" : fieldName.get(0); - throw new CompilationException(ErrorCode.COMPILATION_FIELD_NOT_FOUND, sourceLoc, unTypeField); + throw new CompilationException(ErrorCode.COMPILATION_FIELD_NOT_FOUND, sourceLoc, + LogRedactionUtil.userData(unTypeField)); } partitioningExprTypes.add(fieldType); ATypeTag pkTypeTag = fieldType.getTypeTag(); @@ -144,14 +146,14 @@ public class ValidateUtil { IAType fieldType = partitioningExprTypes.get(i); if (fieldType == null) { throw new CompilationException(ErrorCode.COMPILATION_FIELD_NOT_FOUND, sourceLoc, - RecordUtil.toFullyQualifiedName(partitioningExpr)); + LogRedactionUtil.userData(RecordUtil.toFullyQualifiedName(partitioningExpr))); } boolean nullable = KeyFieldTypeUtil.chooseSource(keySourceIndicators, i, recType, metaRecType) .isSubFieldNullable(partitioningExpr); if (nullable) { // key field is nullable throw new CompilationException(ErrorCode.COMPILATION_PRIMARY_KEY_CANNOT_BE_NULLABLE, sourceLoc, - RecordUtil.toFullyQualifiedName(partitioningExpr)); + LogRedactionUtil.userData(RecordUtil.toFullyQualifiedName(partitioningExpr))); } switch (fieldType.getTypeTag()) { case TINYINT: @@ -171,7 +173,7 @@ public class ValidateUtil { break; case UNION: throw new CompilationException(ErrorCode.COMPILATION_PRIMARY_KEY_CANNOT_BE_NULLABLE, sourceLoc, - RecordUtil.toFullyQualifiedName(partitioningExpr)); + LogRedactionUtil.userData(RecordUtil.toFullyQualifiedName(partitioningExpr))); default: throw new CompilationException(ErrorCode.COMPILATION_ILLEGAL_PRIMARY_KEY_TYPE, sourceLoc, fieldType.getTypeTag()); @@ -209,16 +211,20 @@ public class ValidateUtil { fieldType = keyFieldTypes.get(pos); if (keyFieldTypes.get(pos) == BuiltinType.AMISSING) { throw new CompilationException(ErrorCode.COMPILATION_ERROR, sourceLoc, - "A field with this name \"" + fieldName + "\" could not be found."); + "A field with this name \"" + + LogRedactionUtil.userData(RecordUtil.toFullyQualifiedName(fieldName)) + + "\" could not be found."); } } else if (openFieldCompositeIdx) { - throw new CompilationException(ErrorCode.COMPILATION_ERROR, sourceLoc, "A closed field \"" + fieldName - + "\" could be only in a prefix part of the composite index, containing opened field."); + throw new CompilationException(ErrorCode.COMPILATION_ERROR, sourceLoc, + "A closed field \"" + LogRedactionUtil.userData(RecordUtil.toFullyQualifiedName(fieldName)) + + "\" could be only in a prefix part of the composite index, containing opened field."); } if (keyFieldTypes.get(pos) != BuiltinType.AMISSING && fieldType.getTypeTag() != keyFieldTypes.get(pos).getTypeTag()) { throw new CompilationException(ErrorCode.COMPILATION_ERROR, sourceLoc, - "A field \"" + fieldName + "\" is already defined with the type \"" + fieldType + "\""); + "A field \"" + LogRedactionUtil.userData(RecordUtil.toFullyQualifiedName(fieldName)) + "\" is " + + "already defined with the type \"" + fieldType + "\""); } switch (indexType) { case BTREE: @@ -241,7 +247,9 @@ public class ValidateUtil { break; default: throw new CompilationException(ErrorCode.COMPILATION_ERROR, sourceLoc, - "The field \"" + fieldName + "\" which is of type " + fieldType.getTypeTag() + "The field '" + + LogRedactionUtil.userData(RecordUtil.toFullyQualifiedName(fieldName)) + + "' which is of type " + fieldType.getTypeTag() + " cannot be indexed using the BTree index."); } break; @@ -257,7 +265,9 @@ public class ValidateUtil { break; default: throw new CompilationException(ErrorCode.COMPILATION_ERROR, sourceLoc, - "The field \"" + fieldName + "\" which is of type " + fieldType.getTypeTag() + "The field '" + + LogRedactionUtil.userData(RecordUtil.toFullyQualifiedName(fieldName)) + + "' which is of type " + fieldType.getTypeTag() + " cannot be indexed using the RTree index."); } break; @@ -268,7 +278,9 @@ public class ValidateUtil { break; default: throw new CompilationException(ErrorCode.COMPILATION_ERROR, sourceLoc, - "The field \"" + fieldName + "\" which is of type " + fieldType.getTypeTag() + "The field '" + + LogRedactionUtil.userData(RecordUtil.toFullyQualifiedName(fieldName)) + + "' which is of type " + fieldType.getTypeTag() + " cannot be indexed using the Length Partitioned N-Gram index."); } break; @@ -281,7 +293,9 @@ public class ValidateUtil { break; default: throw new CompilationException(ErrorCode.COMPILATION_ERROR, sourceLoc, - "The field \"" + fieldName + "\" which is of type " + fieldType.getTypeTag() + "The field '" + + LogRedactionUtil.userData(RecordUtil.toFullyQualifiedName(fieldName)) + + "' which is of type " + fieldType.getTypeTag() + " cannot be indexed using the Length Partitioned Keyword index."); } break; @@ -292,7 +306,9 @@ public class ValidateUtil { break; default: throw new CompilationException(ErrorCode.COMPILATION_ERROR, sourceLoc, - "The field \"" + fieldName + "\" which is of type " + fieldType.getTypeTag() + "The field '" + + LogRedactionUtil.userData(RecordUtil.toFullyQualifiedName(fieldName)) + + "' which is of type " + fieldType.getTypeTag() + " cannot be indexed using the N-Gram index."); } break; @@ -305,7 +321,9 @@ public class ValidateUtil { break; default: throw new CompilationException(ErrorCode.COMPILATION_ERROR, sourceLoc, - "The field \"" + fieldName + "\" which is of type " + fieldType.getTypeTag() + "The field '" + + LogRedactionUtil.userData(RecordUtil.toFullyQualifiedName(fieldName)) + + "' which is of type " + fieldType.getTypeTag() + " cannot be indexed using the Keyword index."); } break; diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java index 38225d722a..3aa8a6f399 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java @@ -41,6 +41,7 @@ import java.util.Objects; import java.util.Properties; import java.util.Set; import java.util.concurrent.ExecutorService; +import java.util.stream.Collectors; import org.apache.asterix.active.ActivityState; import org.apache.asterix.active.EntityId; @@ -186,6 +187,7 @@ import org.apache.asterix.om.types.ATypeTag; import org.apache.asterix.om.types.BuiltinType; import org.apache.asterix.om.types.IAType; import org.apache.asterix.om.types.TypeSignature; +import org.apache.asterix.om.utils.RecordUtil; import org.apache.asterix.transaction.management.service.transaction.DatasetIdFactory; import org.apache.asterix.translator.AbstractLangTranslator; import org.apache.asterix.translator.ClientRequest; @@ -236,6 +238,7 @@ import org.apache.hyracks.control.cc.ClusterControllerService; import org.apache.hyracks.control.common.controllers.CCConfig; import org.apache.hyracks.storage.am.common.dataflow.IndexDropOperatorDescriptor.DropOption; import org.apache.hyracks.storage.am.lsm.common.api.ILSMMergePolicyFactory; +import org.apache.hyracks.util.LogRedactionUtil; import org.apache.logging.log4j.Level; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -1090,7 +1093,7 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen } if (stmtCreateIndex.isEnforced() && !fieldExpr.second.isUnknownable()) { throw new AsterixException(ErrorCode.INDEX_ILLEGAL_ENFORCED_NON_OPTIONAL, sourceLoc, - String.valueOf(fieldExpr.first)); + LogRedactionUtil.userData(String.valueOf(fieldExpr.first))); } // don't allow creating an enforced index on a closed-type field, fields that // are part of schema. @@ -1098,11 +1101,13 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen if (stmtCreateIndex.isEnforced() && subType.getSubFieldType(fieldExpr.first.subList(i, fieldExpr.first.size())) != null) { throw new AsterixException(ErrorCode.INDEX_ILLEGAL_ENFORCED_ON_CLOSED_FIELD, sourceLoc, - String.valueOf(fieldExpr.first)); + LogRedactionUtil.userData(String.valueOf(fieldExpr.first))); } if (!isOpen) { - throw new CompilationException(ErrorCode.COMPILATION_ERROR, sourceLoc, "Typed index on \"" - + fieldExpr.first + "\" field could be created only for open datatype"); + throw new CompilationException(ErrorCode.COMPILATION_ERROR, sourceLoc, + "Typed index on '" + + LogRedactionUtil.userData(RecordUtil.toFullyQualifiedName(fieldExpr.first)) + + "' " + "field could be created only for open datatype"); } if (stmtCreateIndex.hasMetaField()) { throw new CompilationException(ErrorCode.COMPILATION_ERROR, sourceLoc, @@ -1115,8 +1120,9 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen overridesFieldTypes = true; } if (fieldType == null) { - throw new CompilationException(ErrorCode.UNKNOWN_TYPE, sourceLoc, fieldExpr.second == null - ? String.valueOf(fieldExpr.first) : String.valueOf(fieldExpr.second)); + throw new CompilationException(ErrorCode.UNKNOWN_TYPE, sourceLoc, + fieldExpr.second == null ? LogRedactionUtil.userData(String.valueOf(fieldExpr.first)) + : String.valueOf(fieldExpr.second)); } // try to add the key & its source to the set of keys, if key couldn't be added, @@ -1124,7 +1130,7 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen if (!indexKeysSet .add(new Pair<>(fieldExpr.first, stmtCreateIndex.getFieldSourceIndicators().get(keyIndex)))) { throw new AsterixException(ErrorCode.INDEX_ILLEGAL_REPETITIVE_FIELD, sourceLoc, - String.valueOf(fieldExpr.first)); + LogRedactionUtil.userData(String.valueOf(fieldExpr.first))); } indexFields.add(fieldExpr.first); @@ -1232,10 +1238,13 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen if (existingIndex.getKeyFieldNames().equals(index.getKeyFieldNames()) && !existingIndex.getKeyFieldTypes().equals(index.getKeyFieldTypes()) && existingIndex.isEnforced()) { - throw new CompilationException(ErrorCode.COMPILATION_ERROR, sourceLoc, "Cannot create index " - + index.getIndexName() + " , enforced index " + existingIndex.getIndexName() - + " on field \"" + StringUtils.join(index.getKeyFieldNames(), ',') - + "\" is already defined with type \"" + existingIndex.getKeyFieldTypes() + "\""); + String fieldNames = index.getKeyFieldNames().stream().map(RecordUtil::toFullyQualifiedName) + .collect(Collectors.joining(",")); + throw new CompilationException(ErrorCode.COMPILATION_ERROR, sourceLoc, + "Cannot create index " + index.getIndexName() + " , enforced index " + + existingIndex.getIndexName() + " on field(s) '" + + LogRedactionUtil.userData(fieldNames) + "' is already defined with type(s) '" + + StringUtils.join(existingIndex.getKeyFieldTypes(), ',') + "'"); } } } diff --git a/asterixdb/asterix-app/src/test/resources/metadata/testsuite.xml b/asterixdb/asterix-app/src/test/resources/metadata/testsuite.xml index 8af4c67d8e..2318d91752 100644 --- a/asterixdb/asterix-app/src/test/resources/metadata/testsuite.xml +++ b/asterixdb/asterix-app/src/test/resources/metadata/testsuite.xml @@ -524,37 +524,37 @@ <test-case FilePath="exception"> <compilation-unit name="issue_384_create_index_error_1"> <output-dir compare="Text">none</output-dir> - <expected-error>ASX1079: Compilation error: The field "[loc]" which is of type point cannot be indexed using the BTree index. (in line 37, at column 1)</expected-error> + <expected-error>ASX1079: Compilation error: The field 'loc' which is of type point cannot be indexed using the BTree index. (in line 37, at column 1)</expected-error> </compilation-unit> </test-case> <test-case FilePath="exception"> <compilation-unit name="issue_384_create_index_error_2"> <output-dir compare="Text">none</output-dir> - <expected-error>ASX1079: Compilation error: The field "[age]" which is of type integer cannot be indexed using the RTree index. (in line 37, at column 1)</expected-error> + <expected-error>ASX1079: Compilation error: The field 'age' which is of type integer cannot be indexed using the RTree index. (in line 37, at column 1)</expected-error> </compilation-unit> </test-case> <test-case FilePath="exception"> <compilation-unit name="issue_384_create_index_error_3"> <output-dir compare="Text">none</output-dir> - <expected-error>ASX1079: Compilation error: The field "[loc]" which is of type point cannot be indexed using the Length Partitioned Keyword index. (in line 37, at column 1)</expected-error> + <expected-error>ASX1079: Compilation error: The field 'loc' which is of type point cannot be indexed using the Length Partitioned Keyword index. (in line 37, at column 1)</expected-error> </compilation-unit> </test-case> <test-case FilePath="exception"> <compilation-unit name="issue_384_create_index_error_4"> <output-dir compare="Text">none</output-dir> - <expected-error>ASX1079: Compilation error: The field "[loc]" which is of type point cannot be indexed using the Length Partitioned Keyword index. (in line 37, at column 1)</expected-error> + <expected-error>ASX1079: Compilation error: The field 'loc' which is of type point cannot be indexed using the Length Partitioned Keyword index. (in line 37, at column 1)</expected-error> </compilation-unit> </test-case> <test-case FilePath="exception"> <compilation-unit name="issue_384_create_index_error_5"> <output-dir compare="Text">none</output-dir> - <expected-error>ASX1079: Compilation error: The field "[loc]" which is of type point cannot be indexed using the Length Partitioned N-Gram index. (in line 37, at column 1)</expected-error> + <expected-error>ASX1079: Compilation error: The field 'loc' which is of type point cannot be indexed using the Length Partitioned N-Gram index. (in line 37, at column 1)</expected-error> </compilation-unit> </test-case> <test-case FilePath="exception"> <compilation-unit name="issue_384_create_index_error_6"> <output-dir compare="Text">none</output-dir> - <expected-error>ASX1079: Compilation error: The field "[loc]" which is of type point cannot be indexed using the Length Partitioned N-Gram index. (in line 37, at column 1)</expected-error> + <expected-error>ASX1079: Compilation error: The field 'loc' which is of type point cannot be indexed using the Length Partitioned N-Gram index. (in line 37, at column 1)</expected-error> </compilation-unit> </test-case> </test-group> diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml index f37648e007..585062bd5c 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml +++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml @@ -7038,19 +7038,19 @@ <test-case FilePath="open-index-enforced/error-checking"> <compilation-unit name="index-on-closed-type"> <output-dir compare="Text">index-on-closed-type</output-dir> - <expected-error>Typed index on "[value]" field could be created only for open datatype</expected-error> + <expected-error>Typed index on 'value' field could be created only for open datatype</expected-error> </compilation-unit> </test-case> <test-case FilePath="open-index-enforced/error-checking"> <compilation-unit name="index-type-collision"> <output-dir compare="Text">index-type-collision</output-dir> - <expected-error>Cannot create index testIdx2 , enforced index testIdx1 on field "[value]" is already defined with type "[integer]"</expected-error> + <expected-error>Cannot create index testIdx2 , enforced index testIdx1 on field(s) 'value' is already defined with type(s) 'integer'</expected-error> </compilation-unit> </test-case> <test-case FilePath="open-index-enforced/error-checking"> <compilation-unit name="index-type-promotion-collision"> <output-dir compare="Text">index-type-promotion-collision</output-dir> - <expected-error>Cannot create index testIdx2 , enforced index testIdx1 on field "[value]" is already defined with type "[bigint]"</expected-error> + <expected-error>Cannot create index testIdx2 , enforced index testIdx1 on field(s) 'value' is already defined with type(s) 'bigint'</expected-error> </compilation-unit> </test-case> <test-case FilePath="open-index-enforced/error-checking"> diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/base/JRecord.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/base/JRecord.java index 9bd6461cb7..622533b5a9 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/base/JRecord.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/base/JRecord.java @@ -40,6 +40,7 @@ import org.apache.asterix.om.types.ATypeTag; import org.apache.asterix.om.types.IAType; import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.data.std.util.ArrayBackedValueStorage; +import org.apache.hyracks.util.LogRedactionUtil; public final class JRecord extends JComplexObject<Map<String, Object>> { @@ -89,7 +90,8 @@ public final class JRecord extends JComplexObject<Map<String, Object>> { // check open part IJObject fieldValue = openFields.get(fieldName); if (fieldValue == null) { - throw new RuntimeDataException(ErrorCode.LIBRARY_JAVA_JOBJECTS_UNKNOWN_FIELD, fieldName); + throw new RuntimeDataException(ErrorCode.LIBRARY_JAVA_JOBJECTS_UNKNOWN_FIELD, + LogRedactionUtil.userData(fieldName)); } return fieldValue; } diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/ADMDataParser.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/ADMDataParser.java index 4a46717588..d75b84df5d 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/ADMDataParser.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/ADMDataParser.java @@ -55,6 +55,7 @@ import org.apache.asterix.runtime.operators.file.adm.AdmLexer.TokenImage; import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.data.std.api.IMutableValueStorage; import org.apache.hyracks.data.std.util.ArrayBackedValueStorage; +import org.apache.hyracks.util.LogRedactionUtil; /** * Parser for ADM formatted data. @@ -472,7 +473,7 @@ public class ADMDataParser extends AbstractDataParser implements IStreamDataPars fieldId = recBuilder.getFieldId(fldName); if ((fieldId < 0) && !recType.isOpen()) { throw new ParseException(ErrorCode.PARSER_ADM_DATA_PARSER_EXTRA_FIELD_IN_CLOSED_RECORD, - fldName); + LogRedactionUtil.userData(fldName)); } else if ((fieldId < 0) && recType.isOpen()) { parseString(tmpTokenImage.getBuffer(), tmpTokenImage.getBegin() + 1, tmpTokenImage.getLength() - 2, fieldNameBuffer.getDataOutput()); @@ -527,7 +528,7 @@ public class ADMDataParser extends AbstractDataParser implements IStreamDataPars final int nullableFieldId = checkOptionalConstraints(recType, nulls); if (nullableFieldId != -1) { throw new ParseException(ErrorCode.PARSER_ADM_DATA_PARSER_FIELD_NOT_NULL, - recType.getFieldNames()[nullableFieldId]); + LogRedactionUtil.userData(recType.getFieldNames()[nullableFieldId])); } } recBuilder.write(out, true); diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/AbstractJsonDataParser.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/AbstractJsonDataParser.java index 2d20cc2304..1dab9eda9d 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/AbstractJsonDataParser.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/AbstractJsonDataParser.java @@ -48,6 +48,7 @@ import org.apache.asterix.runtime.exceptions.UnsupportedTypeException; import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.api.util.ExceptionUtils; import org.apache.hyracks.data.std.api.IMutableValueStorage; +import org.apache.hyracks.util.LogRedactionUtil; import org.apache.hyracks.util.ParseUtil; import com.fasterxml.jackson.core.JsonFactory; @@ -200,7 +201,7 @@ public abstract class AbstractJsonDataParser extends AbstractNestedDataParser<AD if (!recordType.isOpen() && fieldIndex < 0) { throw new RuntimeDataException(ErrorCode.PARSER_ADM_DATA_PARSER_EXTRA_FIELD_IN_CLOSED_RECORD, - fieldName); + LogRedactionUtil.userData(fieldName)); } valueBuffer.reset(); nextToken(); @@ -215,7 +216,8 @@ public abstract class AbstractJsonDataParser extends AbstractNestedDataParser<AD //fail fast if the current field is not nullable if (currentToken() == ADMToken.NULL && !isNullableType(fieldType)) { - throw new RuntimeDataException(ErrorCode.PARSER_EXT_DATA_PARSER_CLOSED_FIELD_NULL, fieldName); + throw new RuntimeDataException(ErrorCode.PARSER_EXT_DATA_PARSER_CLOSED_FIELD_NULL, + LogRedactionUtil.userData(fieldName)); } nullBitMap.set(fieldIndex); diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/AbstractNestedDataParser.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/AbstractNestedDataParser.java index eecbb19f3b..433b949fa2 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/AbstractNestedDataParser.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/AbstractNestedDataParser.java @@ -35,6 +35,7 @@ import org.apache.asterix.om.types.IAType; import org.apache.asterix.om.types.hierachy.ATypeHierarchy; import org.apache.asterix.om.utils.RecordUtil; import org.apache.hyracks.api.exceptions.HyracksDataException; +import org.apache.hyracks.util.LogRedactionUtil; /** * Abstract class for nested formats (ADM, JSON, XML ... etc) @@ -117,7 +118,7 @@ public abstract class AbstractNestedDataParser<T> extends AbstractDataParser { for (int i = 0; i < recordType.getFieldTypes().length; i++) { if (!nullBitmap.get(i) && !isMissableType(recordType.getFieldTypes()[i])) { throw new RuntimeDataException(ErrorCode.PARSER_EXT_DATA_PARSER_CLOSED_FIELD_NULL, - recordType.getFieldNames()[i]); + LogRedactionUtil.userData(recordType.getFieldNames()[i])); } } } diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/DelimitedDataParser.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/DelimitedDataParser.java index 590f51d781..2936d11b80 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/DelimitedDataParser.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/DelimitedDataParser.java @@ -51,6 +51,7 @@ import org.apache.hyracks.data.std.util.ArrayBackedValueStorage; import org.apache.hyracks.dataflow.common.data.parsers.IValueParser; import org.apache.hyracks.dataflow.common.data.parsers.IValueParserFactory; import org.apache.hyracks.dataflow.std.file.FieldCursorForDelimitedDataParser; +import org.apache.hyracks.util.LogRedactionUtil; import org.apache.hyracks.util.ParseUtil; public class DelimitedDataParser extends AbstractDataParser implements IStreamDataParser, IRecordDataParser<char[]> { @@ -108,7 +109,8 @@ public class DelimitedDataParser extends AbstractDataParser implements IStreamDa fldIds[i] = recBuilder.getFieldId(name); if (fldIds[i] < 0) { if (!recordType.isOpen()) { - throw new RuntimeDataException(ErrorCode.PARSER_DELIMITED_ILLEGAL_FIELD, name, recordType); + throw new RuntimeDataException(ErrorCode.PARSER_DELIMITED_ILLEGAL_FIELD, + LogRedactionUtil.userData(name), recordType); } else { nameBuffers[i] = new ArrayBackedValueStorage(); str.setValue(name); diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/TweetParser.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/TweetParser.java index b970b04f62..99d3a5bab5 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/TweetParser.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/TweetParser.java @@ -43,6 +43,7 @@ import org.apache.asterix.om.util.container.ListObjectPool; import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.data.std.api.IMutableValueStorage; import org.apache.hyracks.data.std.util.ArrayBackedValueStorage; +import org.apache.hyracks.util.LogRedactionUtil; import org.apache.hyracks.util.string.UTF8StringWriter; import com.fasterxml.jackson.databind.JsonNode; @@ -200,7 +201,7 @@ public class TweetParser extends AbstractDataParser implements IRecordDataParser if (obj.get(curFNames[iter1]).isNull() && !(curTypes[iter1] instanceof AUnionType)) { if (curRecType.isClosedField(curFNames[iter1])) { throw new RuntimeDataException(ErrorCode.PARSER_EXT_DATA_PARSER_CLOSED_FIELD_NULL, - curFNames[iter1]); + LogRedactionUtil.userData(curFNames[iter1])); } else { continue; } diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/RTreeResourceFactoryProvider.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/RTreeResourceFactoryProvider.java index 2108a4cffe..9cebdc254f 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/RTreeResourceFactoryProvider.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/RTreeResourceFactoryProvider.java @@ -37,7 +37,7 @@ import org.apache.asterix.om.types.ARecordType; import org.apache.asterix.om.types.ATypeTag; import org.apache.asterix.om.types.IAType; import org.apache.asterix.om.utils.NonTaggedFormatUtil; -import org.apache.commons.lang3.StringUtils; +import org.apache.asterix.om.utils.RecordUtil; import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException; import org.apache.hyracks.algebricks.common.utils.Pair; import org.apache.hyracks.algebricks.data.IBinaryComparatorFactoryProvider; @@ -57,6 +57,7 @@ import org.apache.hyracks.storage.am.lsm.rtree.dataflow.LSMRTreeWithAntiMatterLo import org.apache.hyracks.storage.am.rtree.frames.RTreePolicyType; import org.apache.hyracks.storage.common.IResourceFactory; import org.apache.hyracks.storage.common.IStorageManager; +import org.apache.hyracks.util.LogRedactionUtil; public class RTreeResourceFactoryProvider implements IResourceFactoryProvider { @@ -79,7 +80,7 @@ public class RTreeResourceFactoryProvider implements IResourceFactoryProvider { index.getKeyFieldNames().get(0), recordType).first; if (spatialType == null) { throw new CompilationException(ErrorCode.COMPILATION_FIELD_NOT_FOUND, - StringUtils.join(index.getKeyFieldNames().get(0), '.')); + LogRedactionUtil.userData(RecordUtil.toFullyQualifiedName(index.getKeyFieldNames().get(0)))); } List<List<String>> primaryKeyFields = dataset.getPrimaryKeys(); int numPrimaryKeys = primaryKeyFields.size(); diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/TypeUtil.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/TypeUtil.java index 65a800e93f..1f0b487ed7 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/TypeUtil.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/TypeUtil.java @@ -42,6 +42,7 @@ import org.apache.asterix.om.types.hierachy.ATypeHierarchy; import org.apache.commons.lang3.ArrayUtils; import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException; import org.apache.hyracks.algebricks.common.utils.Pair; +import org.apache.hyracks.util.LogRedactionUtil; /** * Provider utility methods for data types @@ -136,8 +137,9 @@ public class TypeUtil { if (enforcedFieldType != null && !ATypeHierarchy.canPromote(enforcedFieldType.getTypeTag(), index.getKeyFieldTypes().get(i).getTypeTag())) { throw new AsterixException(ErrorCode.COMPILATION_ERROR, - "Cannot enforce field \"" + String.join(".", index.getKeyFieldNames().get(i)) - + "\" to have type " + index.getKeyFieldTypes().get(i)); + "Cannot enforce field '" + + LogRedactionUtil.userData(String.join(".", index.getKeyFieldNames().get(i))) + + "' to have type " + index.getKeyFieldTypes().get(i)); } if (enforcedFieldType == null) { recordNameTypesMap.put(splits.get(splits.size() - 1), @@ -220,8 +222,8 @@ public class TypeUtil { IAType actualType = TypeComputeUtils.getActualType(nestedRecordType); if (actualType.getTypeTag() != ATypeTag.OBJECT) { String fName = String.join(".", fieldName); - throw new AsterixException(ErrorCode.COMPILATION_ERROR, - "Field accessor is not defined for \"" + fName + "\" of type " + actualType.getTypeTag()); + throw new AsterixException(ErrorCode.COMPILATION_ERROR, "Field accessor is not defined for '" + + LogRedactionUtil.userData(fName) + "' of type " + actualType.getTypeTag()); } } diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/typecomputer/impl/ClosedRecordConstructorResultType.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/typecomputer/impl/ClosedRecordConstructorResultType.java index 077e9b34dd..d3cb2d7151 100644 --- a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/typecomputer/impl/ClosedRecordConstructorResultType.java +++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/typecomputer/impl/ClosedRecordConstructorResultType.java @@ -38,6 +38,7 @@ import org.apache.hyracks.algebricks.core.algebra.base.LogicalExpressionTag; import org.apache.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression; import org.apache.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment; import org.apache.hyracks.algebricks.core.algebra.metadata.IMetadataProvider; +import org.apache.hyracks.util.LogRedactionUtil; public class ClosedRecordConstructorResultType implements IResultTypeComputer { @@ -76,7 +77,8 @@ public class ClosedRecordConstructorResultType implements IResultTypeComputer { } for (int j = 0; j < i; j++) { if (fieldName.equals(fieldNames[j])) { - throw new CompilationException(ErrorCode.DUPLICATE_FIELD_NAME, f.getSourceLocation(), fieldName); + throw new CompilationException(ErrorCode.DUPLICATE_FIELD_NAME, f.getSourceLocation(), + LogRedactionUtil.userData(fieldName)); } } fieldTypes[i] = e2Type; diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/typecomputer/impl/OpenRecordConstructorResultType.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/typecomputer/impl/OpenRecordConstructorResultType.java index 6deb17c608..fb01764472 100644 --- a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/typecomputer/impl/OpenRecordConstructorResultType.java +++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/typecomputer/impl/OpenRecordConstructorResultType.java @@ -41,6 +41,7 @@ import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression; import org.apache.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression; import org.apache.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment; import org.apache.hyracks.algebricks.core.algebra.metadata.IMetadataProvider; +import org.apache.hyracks.util.LogRedactionUtil; public class OpenRecordConstructorResultType implements IResultTypeComputer { @@ -75,7 +76,8 @@ public class OpenRecordConstructorResultType implements IResultTypeComputer { String fieldName = ConstantExpressionUtil.getStringConstant(e1); if (fieldName != null && t2 != null && TypeHelper.isClosed(t2)) { if (namesList.contains(fieldName)) { - throw new CompilationException(ErrorCode.DUPLICATE_FIELD_NAME, f.getSourceLocation(), fieldName); + throw new CompilationException(ErrorCode.DUPLICATE_FIELD_NAME, f.getSourceLocation(), + LogRedactionUtil.userData(fieldName)); } namesList.add(fieldName); if (t2.getTypeTag() == ATypeTag.UNION) { diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/typecomputer/impl/RecordMergeTypeComputer.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/typecomputer/impl/RecordMergeTypeComputer.java index b1e4021ca3..9985a4cd90 100644 --- a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/typecomputer/impl/RecordMergeTypeComputer.java +++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/typecomputer/impl/RecordMergeTypeComputer.java @@ -40,6 +40,7 @@ import org.apache.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvir import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier; import org.apache.hyracks.algebricks.core.algebra.metadata.IMetadataProvider; import org.apache.hyracks.api.exceptions.SourceLocation; +import org.apache.hyracks.util.LogRedactionUtil; public class RecordMergeTypeComputer implements IResultTypeComputer { @@ -111,7 +112,7 @@ public class RecordMergeTypeComputer implements IResultTypeComputer { // If the ignore duplicates flag is not set, we throw a duplicate field exception else { throw new CompilationException(ErrorCode.COMPILATION_DUPLICATE_FIELD_NAME, - f.getSourceLocation(), fieldNames[i]); + f.getSourceLocation(), LogRedactionUtil.userData(fieldNames[i])); } } @@ -144,7 +145,8 @@ public class RecordMergeTypeComputer implements IResultTypeComputer { private IAType mergedNestedType(String fieldName, IAType fieldType1, IAType fieldType0, SourceLocation sourceLoc) throws AlgebricksException { if (fieldType1.getTypeTag() != ATypeTag.OBJECT || fieldType0.getTypeTag() != ATypeTag.OBJECT) { - throw new CompilationException(ErrorCode.COMPILATION_DUPLICATE_FIELD_NAME, sourceLoc, fieldName); + throw new CompilationException(ErrorCode.COMPILATION_DUPLICATE_FIELD_NAME, sourceLoc, + LogRedactionUtil.userData(fieldName)); } ARecordType resultType = (ARecordType) fieldType0; diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/types/runtime/RuntimeRecordTypeInfo.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/types/runtime/RuntimeRecordTypeInfo.java index ccdfa0ed38..9d53ab3b96 100644 --- a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/types/runtime/RuntimeRecordTypeInfo.java +++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/types/runtime/RuntimeRecordTypeInfo.java @@ -32,6 +32,7 @@ import org.apache.hyracks.data.std.accessors.PointableBinaryHashFunctionFactory; import org.apache.hyracks.data.std.accessors.UTF8StringBinaryComparatorFactory; import org.apache.hyracks.data.std.primitive.UTF8StringPointable; import org.apache.hyracks.data.std.util.ByteArrayAccessibleOutputStream; +import org.apache.hyracks.util.LogRedactionUtil; import org.apache.hyracks.util.string.UTF8StringUtil; import org.apache.hyracks.util.string.UTF8StringWriter; @@ -100,7 +101,8 @@ public class RuntimeRecordTypeInfo { int j = getFieldIndex(baaos.getByteArray(), serializedFieldNameOffsets[i], UTF8StringUtil.getStringLength(baaos.getByteArray(), serializedFieldNameOffsets[i])); if (j != i) { - throw new RuntimeDataException(ErrorCode.DUPLICATE_FIELD_NAME, fieldNames[i]); + throw new RuntimeDataException(ErrorCode.DUPLICATE_FIELD_NAME, + LogRedactionUtil.userData(fieldNames[i])); } } } catch (IOException e) {
