This is an automated email from the ASF dual-hosted git repository. mblow pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit b10bda466519dac2592f1f4ebfc9ba179e281b4b Author: Ali Alsuliman <[email protected]> AuthorDate: Wed Mar 2 18:56:58 2022 +0300 [NO ISSUE][OTH] Redact field name - user model changes: no - storage format changes: no - interface changes: no Details: Change-Id: I360681ff4e1460561eb5b614ce16f7d6519096b2 Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/15523 Integration-Tests: Jenkins <[email protected]> Tested-by: Jenkins <[email protected]> Reviewed-by: Ali Alsuliman <[email protected]> Reviewed-by: Dmitry Lychagin <[email protected]> --- .../asterix/translator/util/ValidateUtil.java | 38 +++++++++++++++------- .../asterix/app/translator/QueryTranslator.java | 17 +++++++--- .../src/test/resources/metadata/testsuite.xml | 12 +++---- .../test/resources/runtimets/testsuite_sqlpp.xml | 6 ++-- .../apache/asterix/lang/common/util/ViewUtil.java | 4 ++- .../apache/asterix/metadata/utils/TypeUtil.java | 17 ++++++---- 6 files changed, 60 insertions(+), 34 deletions(-) diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/util/ValidateUtil.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/util/ValidateUtil.java index a2b26e7..248e6d0 100644 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/util/ValidateUtil.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/util/ValidateUtil.java @@ -36,6 +36,7 @@ import org.apache.asterix.om.types.IAType; import org.apache.asterix.om.utils.RecordUtil; import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException; import org.apache.hyracks.api.exceptions.SourceLocation; +import org.apache.hyracks.util.LogRedactionUtil; /** * A util that can verify if a filter field, a list of partitioning expressions, @@ -73,7 +74,7 @@ public class ValidateUtil { IAType fieldType = itemType.getSubFieldType(filterField); if (fieldType == null) { throw new CompilationException(ErrorCode.COMPILATION_FIELD_NOT_FOUND, sourceLoc, - RecordUtil.toFullyQualifiedName(filterField)); + LogRedactionUtil.userData(RecordUtil.toFullyQualifiedName(filterField))); } switch (fieldType.getTypeTag()) { case TINYINT: @@ -93,7 +94,7 @@ public class ValidateUtil { break; case UNION: throw new CompilationException(ErrorCode.COMPILATION_FILTER_CANNOT_BE_NULLABLE, - RecordUtil.toFullyQualifiedName(filterField)); + LogRedactionUtil.userData(RecordUtil.toFullyQualifiedName(filterField))); default: throw new CompilationException(ErrorCode.COMPILATION_ILLEGAL_FILTER_TYPE, fieldType.getTypeTag().name()); @@ -143,7 +144,8 @@ public class ValidateUtil { IAType fieldType = recType.getSubFieldType(fieldName); if (fieldType == null) { String unTypeField = fieldName.get(0) == null ? "" : fieldName.get(0); - throw new CompilationException(ErrorCode.COMPILATION_FIELD_NOT_FOUND, sourceLoc, unTypeField); + throw new CompilationException(ErrorCode.COMPILATION_FIELD_NOT_FOUND, sourceLoc, + LogRedactionUtil.userData(unTypeField)); } partitioningExprTypes.add(fieldType); ATypeTag pkTypeTag = fieldType.getTypeTag(); @@ -159,7 +161,7 @@ public class ValidateUtil { IAType fieldType = partitioningExprTypes.get(i); if (fieldType == null) { throw new CompilationException(ErrorCode.COMPILATION_FIELD_NOT_FOUND, sourceLoc, - RecordUtil.toFullyQualifiedName(partitioningExpr)); + LogRedactionUtil.userData(RecordUtil.toFullyQualifiedName(partitioningExpr))); } if (forPrimaryKey) { boolean nullable = KeyFieldTypeUtil.chooseSource(keySourceIndicators, i, recType, metaRecType) @@ -167,7 +169,8 @@ public class ValidateUtil { if (nullable) { // key field is nullable throw new CompilationException(ErrorCode.COMPILATION_KEY_CANNOT_BE_NULLABLE, sourceLoc, - keyKindDisplayName, RecordUtil.toFullyQualifiedName(partitioningExpr)); + keyKindDisplayName, + LogRedactionUtil.userData(RecordUtil.toFullyQualifiedName(partitioningExpr))); } } else { fieldType = TypeComputeUtils.getActualType(fieldType); @@ -190,7 +193,8 @@ public class ValidateUtil { break; case UNION: throw new CompilationException(ErrorCode.COMPILATION_KEY_CANNOT_BE_NULLABLE, sourceLoc, - keyKindDisplayName, RecordUtil.toFullyQualifiedName(partitioningExpr)); + keyKindDisplayName, + LogRedactionUtil.userData(RecordUtil.toFullyQualifiedName(partitioningExpr))); default: throw new CompilationException(ErrorCode.COMPILATION_ILLEGAL_KEY_TYPE, sourceLoc, fieldType.getTypeTag(), keyKindDisplayName); @@ -236,7 +240,9 @@ public class ValidateUtil { break; default: throw new CompilationException(ErrorCode.COMPILATION_ERROR, sourceLoc, - "The field '" + displayFieldName + "' which is of type " + fieldType.getTypeTag() + "The field '" + + LogRedactionUtil.userData(RecordUtil.toFullyQualifiedName(displayFieldName)) + + "' which is " + "of type " + fieldType.getTypeTag() + " cannot be indexed using the BTree index."); } break; @@ -251,14 +257,17 @@ public class ValidateUtil { break; default: throw new CompilationException(ErrorCode.COMPILATION_ERROR, sourceLoc, - "The field '" + displayFieldName + "' which is of type " + fieldType.getTypeTag() + "The field '" + + LogRedactionUtil.userData(RecordUtil.toFullyQualifiedName(displayFieldName)) + + "' which is of type " + fieldType.getTypeTag() + " cannot be indexed using the RTree index."); } break; case LENGTH_PARTITIONED_NGRAM_INVIX: if (fieldType.getTypeTag() != ATypeTag.STRING) { throw new CompilationException(ErrorCode.COMPILATION_ERROR, sourceLoc, - "The field '" + displayFieldName + "' which is of type " + fieldType.getTypeTag() + "The field '" + LogRedactionUtil.userData(RecordUtil.toFullyQualifiedName(displayFieldName)) + + "' which is of type " + fieldType.getTypeTag() + " cannot be indexed using the Length Partitioned N-Gram index."); } break; @@ -270,14 +279,17 @@ public class ValidateUtil { break; default: throw new CompilationException(ErrorCode.COMPILATION_ERROR, sourceLoc, - "The field '" + displayFieldName + "' which is of type " + fieldType.getTypeTag() + "The field '" + + LogRedactionUtil.userData(RecordUtil.toFullyQualifiedName(displayFieldName)) + + "' which is of type " + fieldType.getTypeTag() + " cannot be indexed using the Length Partitioned Keyword index."); } break; case SINGLE_PARTITION_NGRAM_INVIX: if (fieldType.getTypeTag() != ATypeTag.STRING) { throw new CompilationException(ErrorCode.COMPILATION_ERROR, sourceLoc, - "The field '" + displayFieldName + "' which is of type " + fieldType.getTypeTag() + "The field '" + LogRedactionUtil.userData(RecordUtil.toFullyQualifiedName(displayFieldName)) + + "' which is of type " + fieldType.getTypeTag() + " cannot be indexed using the N-Gram index."); } break; @@ -289,7 +301,9 @@ public class ValidateUtil { break; default: throw new CompilationException(ErrorCode.COMPILATION_ERROR, sourceLoc, - "The field '" + displayFieldName + "' which is of type " + fieldType.getTypeTag() + "The field '" + + LogRedactionUtil.userData(RecordUtil.toFullyQualifiedName(displayFieldName)) + + "' which is of type " + fieldType.getTypeTag() + " cannot be indexed using the Keyword index."); } break; diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java index 06458ce..5e4b730 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java @@ -203,6 +203,7 @@ import org.apache.asterix.om.types.ATypeTag; import org.apache.asterix.om.types.BuiltinType; import org.apache.asterix.om.types.IAType; import org.apache.asterix.om.types.TypeSignature; +import org.apache.asterix.om.utils.RecordUtil; import org.apache.asterix.runtime.fulltext.AbstractFullTextFilterDescriptor; import org.apache.asterix.runtime.fulltext.FullTextConfigDescriptor; import org.apache.asterix.runtime.fulltext.IFullTextFilterDescriptor; @@ -259,6 +260,7 @@ import org.apache.hyracks.control.common.controllers.CCConfig; import org.apache.hyracks.storage.am.common.dataflow.IndexDropOperatorDescriptor.DropOption; import org.apache.hyracks.storage.am.lsm.common.api.ILSMMergePolicyFactory; import org.apache.hyracks.storage.am.lsm.invertedindex.fulltext.TokenizerCategory; +import org.apache.hyracks.util.LogRedactionUtil; import org.apache.hyracks.util.OptionalBoolean; import org.apache.logging.log4j.Level; import org.apache.logging.log4j.LogManager; @@ -1223,7 +1225,7 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen if (stmtCreateIndex.hasCastDefaultNull()) { throw new CompilationException(ErrorCode.COMPILATION_ERROR, stmtCreateIndex.getSourceLocation(), - "CAST modifier is used without specifying " + "the type of the indexed field"); + "CAST modifier is used without specifying the type of the indexed field"); } fieldTypePrime = projectTypePrime; fieldTypeNullable = projectTypeNullable; @@ -1250,7 +1252,10 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen // allow overriding the type of the closed-field only if CAST modifier is used if (!stmtCreateIndex.hasCastDefaultNull()) { throw new CompilationException(ErrorCode.COMPILATION_ERROR, - indexedElement.getSourceLocation(), "Typed index on '" + projectPath + indexedElement.getSourceLocation(), + "Typed index on '" + + LogRedactionUtil + .userData(RecordUtil.toFullyQualifiedName(projectPath)) + "' field could be created only for open datatype"); } } @@ -1631,11 +1636,13 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen } if (existingIndexKeyFieldNames.equals(indexKeyFieldNames) && !existingIndexKeyFieldTypes.equals(indexKeyFieldTypes)) { + String fieldNames = indexKeyFieldNames.stream().map(RecordUtil::toFullyQualifiedName) + .collect(Collectors.joining(",")); throw new CompilationException(ErrorCode.COMPILATION_ERROR, sourceLoc, "Cannot create index " + index.getIndexName() + " , enforced index " - + existingIndex.getIndexName() + " on field '" - + StringUtils.join(indexKeyFieldNames, ',') + "' is already defined with type '" - + existingIndexKeyFieldTypes + "'"); + + existingIndex.getIndexName() + " on field(s) '" + + LogRedactionUtil.userData(fieldNames) + "' is already defined with type(s) '" + + StringUtils.join(existingIndexKeyFieldTypes, ',') + "'"); } } } diff --git a/asterixdb/asterix-app/src/test/resources/metadata/testsuite.xml b/asterixdb/asterix-app/src/test/resources/metadata/testsuite.xml index e642922..49e4d74 100644 --- a/asterixdb/asterix-app/src/test/resources/metadata/testsuite.xml +++ b/asterixdb/asterix-app/src/test/resources/metadata/testsuite.xml @@ -530,37 +530,37 @@ <test-case FilePath="exception"> <compilation-unit name="issue_384_create_index_error_1"> <output-dir compare="Text">none</output-dir> - <expected-error>ASX1079: Compilation error: The field '[loc]' which is of type point cannot be indexed using the BTree index. (in line 37, at column 33)</expected-error> + <expected-error>ASX1079: Compilation error: The field 'loc' which is of type point cannot be indexed using the BTree index. (in line 37, at column 33)</expected-error> </compilation-unit> </test-case> <test-case FilePath="exception"> <compilation-unit name="issue_384_create_index_error_2"> <output-dir compare="Text">none</output-dir> - <expected-error>ASX1079: Compilation error: The field '[age]' which is of type integer cannot be indexed using the RTree index. (in line 37, at column 33)</expected-error> + <expected-error>ASX1079: Compilation error: The field 'age' which is of type integer cannot be indexed using the RTree index. (in line 37, at column 33)</expected-error> </compilation-unit> </test-case> <test-case FilePath="exception"> <compilation-unit name="issue_384_create_index_error_3"> <output-dir compare="Text">none</output-dir> - <expected-error>ASX1079: Compilation error: The field '[loc]' which is of type point cannot be indexed using the Length Partitioned Keyword index. (in line 37, at column 33)</expected-error> + <expected-error>ASX1079: Compilation error: The field 'loc' which is of type point cannot be indexed using the Length Partitioned Keyword index. (in line 37, at column 33)</expected-error> </compilation-unit> </test-case> <test-case FilePath="exception"> <compilation-unit name="issue_384_create_index_error_4"> <output-dir compare="Text">none</output-dir> - <expected-error>ASX1079: Compilation error: The field '[loc]' which is of type point cannot be indexed using the Length Partitioned Keyword index. (in line 37, at column 33)</expected-error> + <expected-error>ASX1079: Compilation error: The field 'loc' which is of type point cannot be indexed using the Length Partitioned Keyword index. (in line 37, at column 33)</expected-error> </compilation-unit> </test-case> <test-case FilePath="exception"> <compilation-unit name="issue_384_create_index_error_5"> <output-dir compare="Text">none</output-dir> - <expected-error>ASX1079: Compilation error: The field '[loc]' which is of type point cannot be indexed using the Length Partitioned N-Gram index. (in line 37, at column 33)</expected-error> + <expected-error>ASX1079: Compilation error: The field 'loc' which is of type point cannot be indexed using the Length Partitioned N-Gram index. (in line 37, at column 33)</expected-error> </compilation-unit> </test-case> <test-case FilePath="exception"> <compilation-unit name="issue_384_create_index_error_6"> <output-dir compare="Text">none</output-dir> - <expected-error>ASX1079: Compilation error: The field '[loc]' which is of type point cannot be indexed using the Length Partitioned N-Gram index. (in line 37, at column 33)</expected-error> + <expected-error>ASX1079: Compilation error: The field 'loc' which is of type point cannot be indexed using the Length Partitioned N-Gram index. (in line 37, at column 33)</expected-error> </compilation-unit> </test-case> </test-group> diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml index 14322f2..1267995 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml +++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml @@ -4335,7 +4335,7 @@ <expected-error>CAST modifier is only allowed for B-Tree indexes</expected-error> <expected-error>CAST modifier cannot be specified together with ENFORCED</expected-error> <expected-error>CAST modifier is used without specifying the type of the indexed field</expected-error> - <expected-error>Typed index on '[typed_f2]' field could be created only for open datatype</expected-error> + <expected-error>Typed index on 'typed_f2' field could be created only for open datatype</expected-error> <expected-error>Parameter invalid_date cannot be set</expected-error> </compilation-unit> </test-case> @@ -7326,13 +7326,13 @@ <test-case FilePath="open-index-enforced/error-checking"> <compilation-unit name="index-type-collision"> <output-dir compare="Text">index-type-collision</output-dir> - <expected-error>Cannot create index testIdx2 , enforced index testIdx1 on field '[value]' is already defined with type '[integer]'</expected-error> + <expected-error>Cannot create index testIdx2 , enforced index testIdx1 on field(s) 'value' is already defined with type(s) 'integer'</expected-error> </compilation-unit> </test-case> <test-case FilePath="open-index-enforced/error-checking"> <compilation-unit name="index-type-promotion-collision"> <output-dir compare="Text">index-type-promotion-collision</output-dir> - <expected-error>Cannot create index testIdx2 , enforced index testIdx1 on field '[value]' is already defined with type '[bigint]'</expected-error> + <expected-error>Cannot create index testIdx2 , enforced index testIdx1 on field(s) 'value' is already defined with type(s) 'bigint'</expected-error> </compilation-unit> </test-case> <test-case FilePath="open-index-enforced/error-checking"> diff --git a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/ViewUtil.java b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/ViewUtil.java index cb704b2..975a387 100644 --- a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/ViewUtil.java +++ b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/ViewUtil.java @@ -54,6 +54,7 @@ import org.apache.hyracks.algebricks.common.utils.Triple; import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier; import org.apache.hyracks.api.exceptions.IWarningCollector; import org.apache.hyracks.api.exceptions.SourceLocation; +import org.apache.hyracks.util.LogRedactionUtil; public final class ViewUtil { @@ -133,7 +134,8 @@ public final class ViewUtil { AUnionType unionType = (AUnionType) fieldType; if (!unionType.isNullableType()) { throw new CompilationException(ErrorCode.COMPILATION_ERROR, sourceLoc, - String.format("Invalid type for field %s. Optional type must allow NULL", fieldNames[i])); + String.format("Invalid type for field %s. Optional type must allow NULL", + LogRedactionUtil.userData(fieldNames[i]))); } primeType = unionType.getActualType(); } else { diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/TypeUtil.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/TypeUtil.java index 5201a6a..6e62262 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/TypeUtil.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/TypeUtil.java @@ -54,6 +54,7 @@ import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression; import org.apache.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression; import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier; import org.apache.hyracks.api.exceptions.SourceLocation; +import org.apache.hyracks.util.LogRedactionUtil; /** * Provider utility methods for data types @@ -225,7 +226,8 @@ public class TypeUtil { if (typeIntermediate == null) { String fName = String.join(".", subFieldName); throw new AsterixException(ErrorCode.COMPILATION_ERROR, - "No list item type found. Wrong type given from field " + fName); + "No list item type found. Wrong type given from field " + + LogRedactionUtil.userData(fName)); } subFieldName.add(keyFieldNames.get(i)); } @@ -243,8 +245,8 @@ public class TypeUtil { ATypeTag tt = TypeComputeUtils.getActualType(typeIntermediate).getTypeTag(); if (tt != ATypeTag.OBJECT && tt != ATypeTag.ARRAY && tt != ATypeTag.MULTISET) { String fName = String.join(".", subFieldName); - throw new AsterixException(ErrorCode.COMPILATION_ERROR, - "Field accessor is not defined for \"" + fName + "\" of type " + tt); + throw new AsterixException(ErrorCode.COMPILATION_ERROR, "Field accessor is not defined for '" + + LogRedactionUtil.userData(fName) + "' of type " + tt); } } @@ -309,8 +311,9 @@ public class TypeUtil { recordNameTypesMap.put(keyFieldNames.get(keyFieldNames.size() - 1), AUnionType.createNullableType(nestArrayType(keyFieldType, isKeyTypeWithUnnest))); } else if (!ATypeHierarchy.canPromote(enforcedFieldType.getTypeTag(), this.keyFieldType.getTypeTag())) { - throw new AsterixException(ErrorCode.COMPILATION_ERROR, "Cannot enforce field \"" - + String.join(".", this.keyFieldNames) + "\" to have type " + this.keyFieldType); + throw new AsterixException(ErrorCode.COMPILATION_ERROR, + "Cannot enforce field '" + LogRedactionUtil.userData(String.join(".", this.keyFieldNames)) + + "' to have type " + this.keyFieldType); } } else { recordNameTypesMap.put(keyFieldNames.get(keyFieldNames.size() - 1), @@ -533,8 +536,8 @@ public class TypeUtil { IAType actualType = TypeComputeUtils.getActualType(nestedRecordType); if (actualType.getTypeTag() != ATypeTag.OBJECT) { String fName = String.join(".", fieldName); - throw new AsterixException(ErrorCode.COMPILATION_ERROR, - "Field accessor is not defined for \"" + fName + "\" of type " + actualType.getTypeTag()); + throw new AsterixException(ErrorCode.COMPILATION_ERROR, "Field accessor is not defined for '" + + LogRedactionUtil.userData(fName) + "' of type " + actualType.getTypeTag()); } }
