Ali Alsuliman has submitted this change and it was merged. Change subject: [NO ISSUE][OTH] Support log redaction ......................................................................
[NO ISSUE][OTH] Support log redaction - user model changes: no - storage format changes: no - interface changes: no Details: Support log redaction. Change-Id: I602c833ba2a055da8fbe8782ec62be683ff4581b Reviewed-on: https://asterix-gerrit.ics.uci.edu/3105 Sonar-Qube: Jenkins <[email protected]> Tested-by: Jenkins <[email protected]> Integration-Tests: Jenkins <[email protected]> Contrib: Jenkins <[email protected]> Reviewed-by: Murtadha Hubail <[email protected]> --- M asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/CheckFilterExpressionTypeRule.java M asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/LoadRecordFieldsRule.java M asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/PushFieldAccessRule.java M asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SetAsterixPhysicalOperatorsRule.java M asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SetClosedRecordConstructorsRule.java M asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SweepIllegalNonfunctionalFunctions.java M asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/BTreeAccessMethod.java M asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/FeedLogManager.java M asterixdb/asterix-lang-sqlpp/pom.xml M asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/SqlppQueryRewriter.java M asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/util/ExpressionToVariableUtil.java M asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/PointableHelper.java M hyracks-fullstack/algebricks/algebricks-core/pom.xml M hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/AbstractLogicalOperator.java M hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/FDsAndEquivClassesVisitor.java M hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/AbstractUnnestPOperator.java M hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/jobgen/impl/JobBuilder.java M hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/rewriter/base/AbstractRuleController.java M hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/rewriter/base/HeuristicOptimizer.java M hyracks-fullstack/algebricks/algebricks-rewriter/pom.xml M hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/EnforceStructuralPropertiesRule.java M hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/SetAlgebricksPhysicalOperatorsRule.java M hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/util/JoinUtils.java M hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/ErrorCode.java M hyracks-fullstack/hyracks/hyracks-api/src/main/resources/errormsg/en.properties M hyracks-fullstack/hyracks/hyracks-dataflow-common/src/main/java/org/apache/hyracks/dataflow/common/comm/io/FrameDeserializer.java M hyracks-fullstack/hyracks/hyracks-dataflow-common/src/main/java/org/apache/hyracks/dataflow/common/comm/io/SerializingDataWriter.java A hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/ILogRedactor.java A hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/LogRedactionUtil.java 30 files changed, 192 insertions(+), 79 deletions(-) Approvals: Anon. E. Moose #1000171: Jenkins: Verified; No violations found; ; Verified Murtadha Hubail: Looks good to me, approved diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/CheckFilterExpressionTypeRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/CheckFilterExpressionTypeRule.java index 1eb3e75..4839ac3 100644 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/CheckFilterExpressionTypeRule.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/CheckFilterExpressionTypeRule.java @@ -35,6 +35,7 @@ import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.SelectOperator; import org.apache.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule; +import org.apache.hyracks.util.LogRedactionUtil; /** * This rule is to check if all the filter expression are of the boolean type or a possible (determined @@ -65,7 +66,8 @@ if (condType.getTypeTag() != ATypeTag.BOOLEAN && condType.getTypeTag() != ATypeTag.ANY && !isPossibleBoolean(condType)) { throw new CompilationException(ErrorCode.COMPILATION_ERROR, condition.getSourceLocation(), - "The select condition " + condition.toString() + " should be of the boolean type."); + "The select condition " + LogRedactionUtil.userData(condition.toString()) + " should be of the " + + "boolean type."); } return false; } diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/LoadRecordFieldsRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/LoadRecordFieldsRule.java index 4d61307..6762e77 100644 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/LoadRecordFieldsRule.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/LoadRecordFieldsRule.java @@ -223,7 +223,7 @@ } } throw new CompilationException(ErrorCode.COMPILATION_ERROR, a2.getSourceLocation(), - "Field access " + getFirstExpr(a2) + " does not correspond to any input of operator " + topOp); + "Field access " + getFirstExpr(a2) + " does not correspond to any input"); } } } diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/PushFieldAccessRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/PushFieldAccessRule.java index e6f5d91..421b00d 100644 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/PushFieldAccessRule.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/PushFieldAccessRule.java @@ -64,7 +64,6 @@ import org.apache.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.NestedTupleSourceOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities; -import org.apache.hyracks.algebricks.core.algebra.util.OperatorManipulationUtil; import org.apache.hyracks.algebricks.core.algebra.util.OperatorPropertiesUtil; import org.apache.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule; @@ -283,8 +282,7 @@ } } throw new CompilationException(ErrorCode.COMPILATION_ERROR, assignOp.getSourceLocation(), - "Field access " + assignOp.getExpressions().get(0).getValue() - + " does not correspond to any input of operator " + inputOp); + "Field access " + assignOp.getExpressions().get(0).getValue() + " doesn't correspond to any input"); } else { // check if the accessed field is one of the partitioning key fields. If yes, we can equate the 2 variables if (inputOp.getOperatorTag() == LogicalOperatorTag.DATASOURCESCAN) { diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SetAsterixPhysicalOperatorsRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SetAsterixPhysicalOperatorsRule.java index 2a42da1..cd75c1e 100644 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SetAsterixPhysicalOperatorsRule.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SetAsterixPhysicalOperatorsRule.java @@ -332,10 +332,11 @@ .createMergeAggregation(aggProducedVars.get(i), aggFuncExpr, context); if (mergeExpr == null) { throw new CompilationException(ErrorCode.COMPILATION_ERROR, aggFuncExpr.getSourceLocation(), - "The aggregation function " + aggFuncExpr + "The aggregation function " + + ((AbstractFunctionCallExpression) aggFuncExpr).getFunctionIdentifier().getName() + " does not have a registered intermediate aggregation function."); } - mergeExpressionRefs.add(new MutableObject<ILogicalExpression>(mergeExpr)); + mergeExpressionRefs.add(new MutableObject<>(mergeExpr)); } aggOp.setMergeExpressions(mergeExpressionRefs); } diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SetClosedRecordConstructorsRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SetClosedRecordConstructorsRule.java index 52858e5..9ca80e5 100644 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SetClosedRecordConstructorsRule.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SetClosedRecordConstructorsRule.java @@ -18,6 +18,8 @@ */ package org.apache.asterix.optimizer.rules; +import static org.apache.asterix.common.config.GlobalConfig.ASTERIX_LOGGER; + import org.apache.asterix.common.config.GlobalConfig; import org.apache.asterix.common.exceptions.CompilationException; import org.apache.asterix.common.exceptions.ErrorCode; @@ -42,6 +44,7 @@ import org.apache.hyracks.algebricks.core.algebra.visitors.AbstractConstVarFunVisitor; import org.apache.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform; import org.apache.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule; +import org.apache.hyracks.util.LogRedactionUtil; /** * open-record-constructor() becomes closed-record-constructor() if all the @@ -113,7 +116,8 @@ int n = expr.getArguments().size(); if (n % 2 > 0) { throw new CompilationException(ErrorCode.COMPILATION_ERROR, expr.getSourceLocation(), - "Record constructor expected to have an even number of arguments: " + expr); + "Record constructor expected to have an even number of arguments: " + + LogRedactionUtil.userData(expr.toString())); } for (int i = 0; i < n / 2; i++) { ILogicalExpression a0 = expr.getArguments().get(2 * i).getValue(); @@ -133,7 +137,8 @@ } if (allClosed) { expr.setFunctionInfo(FunctionUtil.getFunctionInfo(BuiltinFunctions.CLOSED_RECORD_CONSTRUCTOR)); - GlobalConfig.ASTERIX_LOGGER.trace("Switching to CLOSED record constructor in " + expr + ".\n"); + ASTERIX_LOGGER.trace(() -> "Switching to CLOSED record constructor in " + + LogRedactionUtil.userData(expr.toString()) + ".\n"); changed = true; } } diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SweepIllegalNonfunctionalFunctions.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SweepIllegalNonfunctionalFunctions.java index 405ff73..5d1d690 100644 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SweepIllegalNonfunctionalFunctions.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SweepIllegalNonfunctionalFunctions.java @@ -101,23 +101,23 @@ private class IllegalNonfunctionalFunctionSweeperOperatorVisitor implements ILogicalOperatorVisitor<Void, Void> { - private void sweepExpression(ILogicalExpression expr, ILogicalOperator op) throws AlgebricksException { + private void sweepExpression(ILogicalExpression expr) throws AlgebricksException { if (expr.getExpressionTag() == LogicalExpressionTag.FUNCTION_CALL && !expr.isFunctional()) { AbstractFunctionCallExpression fce = (AbstractFunctionCallExpression) expr; throw new CompilationException(ErrorCode.COMPILATION_ERROR, fce.getSourceLocation(), - "Found non-functional function " + fce.getFunctionIdentifier() + " in op " + op); + "Found non-functional function " + fce.getFunctionIdentifier()); } } @Override public Void visitAggregateOperator(AggregateOperator op, Void arg) throws AlgebricksException { for (Mutable<ILogicalExpression> me : op.getExpressions()) { - sweepExpression(me.getValue(), op); + sweepExpression(me.getValue()); } List<Mutable<ILogicalExpression>> mergeExprs = op.getMergeExpressions(); if (mergeExprs != null) { for (Mutable<ILogicalExpression> me : mergeExprs) { - sweepExpression(me.getValue(), op); + sweepExpression(me.getValue()); } } return null; @@ -126,7 +126,7 @@ @Override public Void visitRunningAggregateOperator(RunningAggregateOperator op, Void arg) throws AlgebricksException { for (Mutable<ILogicalExpression> me : op.getExpressions()) { - sweepExpression(me.getValue(), op); + sweepExpression(me.getValue()); } return null; } @@ -139,10 +139,10 @@ @Override public Void visitGroupByOperator(GroupByOperator op, Void arg) throws AlgebricksException { for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : op.getGroupByList()) { - sweepExpression(p.second.getValue(), op); + sweepExpression(p.second.getValue()); } for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : op.getDecorList()) { - sweepExpression(p.second.getValue(), op); + sweepExpression(p.second.getValue()); } return null; } @@ -154,13 +154,13 @@ @Override public Void visitInnerJoinOperator(InnerJoinOperator op, Void arg) throws AlgebricksException { - sweepExpression(op.getCondition().getValue(), op); + sweepExpression(op.getCondition().getValue()); return null; } @Override public Void visitLeftOuterJoinOperator(LeftOuterJoinOperator op, Void arg) throws AlgebricksException { - sweepExpression(op.getCondition().getValue(), op); + sweepExpression(op.getCondition().getValue()); return null; } @@ -172,7 +172,7 @@ @Override public Void visitOrderOperator(OrderOperator op, Void arg) throws AlgebricksException { for (Pair<IOrder, Mutable<ILogicalExpression>> p : op.getOrderExpressions()) { - sweepExpression(p.second.getValue(), op); + sweepExpression(p.second.getValue()); } return null; } @@ -266,7 +266,7 @@ @Override public Void visitDistinctOperator(DistinctOperator op, Void arg) throws AlgebricksException { for (Mutable<ILogicalExpression> expr : op.getExpressions()) { - sweepExpression(expr.getValue(), op); + sweepExpression(expr.getValue()); } return null; } @@ -310,40 +310,40 @@ @Override public Void visitForwardOperator(ForwardOperator op, Void arg) throws AlgebricksException { - sweepExpression(op.getRangeMapExpression().getValue(), op); + sweepExpression(op.getRangeMapExpression().getValue()); return null; } @Override public Void visitWindowOperator(WindowOperator op, Void arg) throws AlgebricksException { for (Mutable<ILogicalExpression> me : op.getPartitionExpressions()) { - sweepExpression(me.getValue(), op); + sweepExpression(me.getValue()); } for (Pair<IOrder, Mutable<ILogicalExpression>> p : op.getOrderExpressions()) { - sweepExpression(p.second.getValue(), op); + sweepExpression(p.second.getValue()); } for (Pair<IOrder, Mutable<ILogicalExpression>> p : op.getFrameValueExpressions()) { - sweepExpression(p.second.getValue(), op); + sweepExpression(p.second.getValue()); } for (Mutable<ILogicalExpression> me : op.getFrameStartExpressions()) { - sweepExpression(me.getValue(), op); + sweepExpression(me.getValue()); } for (Mutable<ILogicalExpression> me : op.getFrameEndExpressions()) { - sweepExpression(me.getValue(), op); + sweepExpression(me.getValue()); } for (Mutable<ILogicalExpression> me : op.getFrameExcludeExpressions()) { - sweepExpression(me.getValue(), op); + sweepExpression(me.getValue()); } ILogicalExpression frameOffset = op.getFrameOffset().getValue(); if (frameOffset != null) { - sweepExpression(frameOffset, op); + sweepExpression(frameOffset); } for (Mutable<ILogicalExpression> me : op.getExpressions()) { ILogicalExpression expr = me.getValue(); if (isStatefulFunctionCall(expr)) { for (Mutable<ILogicalExpression> fcallArg : ((AbstractFunctionCallExpression) expr) .getArguments()) { - sweepExpression(fcallArg.getValue(), op); + sweepExpression(fcallArg.getValue()); } } else { throw new CompilationException(ErrorCode.COMPILATION_ILLEGAL_STATE, op.getSourceLocation()); diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/BTreeAccessMethod.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/BTreeAccessMethod.java index 099f769..8d6f04d 100644 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/BTreeAccessMethod.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/BTreeAccessMethod.java @@ -74,6 +74,7 @@ import org.apache.hyracks.algebricks.core.algebra.operators.logical.SelectOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.UnnestMapOperator; import org.apache.hyracks.algebricks.core.algebra.util.OperatorManipulationUtil; +import org.apache.hyracks.util.LogRedactionUtil; /** * Class for helping rewrite rules to choose and apply BTree indexes. @@ -787,7 +788,7 @@ // The original select cond must be an AND. Check it just to be sure. if (funcExpr.getFunctionIdentifier() != AlgebricksBuiltinFunctions.AND) { throw new CompilationException(ErrorCode.COMPILATION_FUNC_EXPRESSION_CANNOT_UTILIZE_INDEX, - funcExpr.getSourceLocation(), funcExpr.toString()); + funcExpr.getSourceLocation(), LogRedactionUtil.userData(funcExpr.toString())); } // Clean the conjuncts. for (Mutable<ILogicalExpression> arg : funcExpr.getArguments()) { diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java index 7c42e5c..99df372 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java @@ -76,6 +76,7 @@ import org.apache.hyracks.http.api.IServletRequest; import org.apache.hyracks.http.api.IServletResponse; import org.apache.hyracks.http.server.utils.HttpUtil; +import org.apache.hyracks.util.LogRedactionUtil; import org.apache.logging.log4j.Level; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -520,7 +521,7 @@ optionalParams = optionalParamProvider.apply(request); } setRequestParam(request, param, optionalParams); - LOGGER.info("handleRequest: {}", param); + LOGGER.info(() -> "handleRequest: " + LogRedactionUtil.userData(param.toString())); ResultDelivery delivery = parseResultDelivery(param.getMode()); setSessionConfig(sessionOutput, param, delivery); final ResultProperties resultProperties = param.getMaxResultReads() == null ? new ResultProperties(delivery) @@ -624,33 +625,38 @@ if (t instanceof org.apache.asterix.aqlplus.parser.TokenMgrError || t instanceof TokenMgrError || t instanceof AlgebricksException) { if (LOGGER.isDebugEnabled()) { - LOGGER.debug("handleException: {}: {}", t.getMessage(), param, t); + LOGGER.debug("handleException: {}: {}", t.getMessage(), LogRedactionUtil.userData(param.toString()), t); } else { - LOGGER.info("handleException: {}: {}", t.getMessage(), param); + LOGGER.info(() -> "handleException: " + t.getMessage() + ": " + + LogRedactionUtil.userData(param.toString())); } state.setStatus(ResultStatus.FATAL, HttpResponseStatus.BAD_REQUEST); } else if (t instanceof HyracksException) { HyracksException he = (HyracksException) t; switch (he.getComponent() + he.getErrorCode()) { case ASTERIX + REQUEST_TIMEOUT: - LOGGER.info("handleException: request execution timed out: {}", param); + LOGGER.info(() -> "handleException: request execution timed out: " + + LogRedactionUtil.userData(param.toString())); state.setStatus(ResultStatus.TIMEOUT, HttpResponseStatus.OK); break; case ASTERIX + REJECT_BAD_CLUSTER_STATE: case ASTERIX + REJECT_NODE_UNREGISTERED: - LOGGER.warn("handleException: {}: {}", he.getMessage(), param); + LOGGER.warn(() -> "handleException: " + he.getMessage() + ": " + + LogRedactionUtil.userData(param.toString())); state.setStatus(ResultStatus.FATAL, HttpResponseStatus.SERVICE_UNAVAILABLE); break; case ASTERIX + NO_STATEMENT_PROVIDED: state.setStatus(ResultStatus.FATAL, HttpResponseStatus.BAD_REQUEST); break; default: - LOGGER.warn("handleException: unexpected exception {}: {}", he.getMessage(), param, he); + LOGGER.warn(() -> "handleException: unexpected exception " + he.getMessage() + ": " + + LogRedactionUtil.userData(param.toString()), he); state.setStatus(ResultStatus.FATAL, HttpResponseStatus.INTERNAL_SERVER_ERROR); break; } } else { - LOGGER.warn("handleException: unexpected exception: {}", param, t); + LOGGER.warn(() -> "handleException: unexpected exception: " + LogRedactionUtil.userData(param.toString()), + t); state.setStatus(ResultStatus.FATAL, HttpResponseStatus.INTERNAL_SERVER_ERROR); } } diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/FeedLogManager.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/FeedLogManager.java index 57c9e2c..282b536 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/FeedLogManager.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/FeedLogManager.java @@ -35,6 +35,7 @@ import org.apache.commons.io.FileUtils; import org.apache.hyracks.api.exceptions.HyracksDataException; +import org.apache.hyracks.util.LogRedactionUtil; public class FeedLogManager implements Closeable { @@ -45,13 +46,13 @@ SNAPSHOT // an identifier that partitions with identifiers before this one should be ignored } - public static final String PROGRESS_LOG_FILE_NAME = "progress.log"; - public static final String ERROR_LOG_FILE_NAME = "error.log"; - public static final String BAD_RECORDS_FILE_NAME = "failed_record.log"; - public static final String START_PREFIX = "s:"; - public static final String END_PREFIX = "e:"; + private static final String PROGRESS_LOG_FILE_NAME = "progress.log"; + private static final String ERROR_LOG_FILE_NAME = "error.log"; + private static final String BAD_RECORDS_FILE_NAME = "failed_record.log"; + private static final String START_PREFIX = "s:"; + private static final String END_PREFIX = "e:"; private static final String DATE_FORMAT_STRING = "MM/dd/yyyy HH:mm:ss"; - public static final int PREFIX_SIZE = START_PREFIX.length() + DATE_FORMAT_STRING.length() + 1; + private static final int PREFIX_SIZE = START_PREFIX.length() + DATE_FORMAT_STRING.length() + 1; private String currentPartition; private final TreeSet<String> completed; private final Path dir; @@ -149,7 +150,7 @@ return true; } - public synchronized void logProgress(String log) throws IOException { + private synchronized void logProgress(String log) throws IOException { stringBuilder.setLength(0); stringBuilder.append(df.format((new Date()))); stringBuilder.append(' '); @@ -173,7 +174,7 @@ public synchronized void logRecord(String record, String errorMessage) throws IOException { stringBuilder.setLength(0); - stringBuilder.append(record); + stringBuilder.append(LogRedactionUtil.userData(record)); stringBuilder.append(ExternalDataConstants.LF); stringBuilder.append(df.format((new Date()))); stringBuilder.append(' '); @@ -183,7 +184,7 @@ recordLogger.flush(); } - public static String getSplitId(String log) { + private static String getSplitId(String log) { return log.substring(PREFIX_SIZE); } diff --git a/asterixdb/asterix-lang-sqlpp/pom.xml b/asterixdb/asterix-lang-sqlpp/pom.xml index dbfe80b..f13c219 100644 --- a/asterixdb/asterix-lang-sqlpp/pom.xml +++ b/asterixdb/asterix-lang-sqlpp/pom.xml @@ -137,6 +137,10 @@ <dependencies> <dependency> + <groupId>org.apache.hyracks</groupId> + <artifactId>hyracks-util</artifactId> + </dependency> + <dependency> <groupId>org.apache.asterix</groupId> <artifactId>asterix-lang-common</artifactId> <version>${project.version}</version> diff --git a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/SqlppQueryRewriter.java b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/SqlppQueryRewriter.java index 35de051..1bed469 100644 --- a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/SqlppQueryRewriter.java +++ b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/SqlppQueryRewriter.java @@ -20,21 +20,18 @@ import java.util.ArrayList; import java.util.Collection; +import java.util.HashSet; import java.util.List; import java.util.Set; -import java.util.HashSet; - -import org.apache.asterix.lang.common.base.AbstractClause; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; import org.apache.asterix.common.exceptions.CompilationException; import org.apache.asterix.common.functions.FunctionSignature; +import org.apache.asterix.lang.common.base.AbstractClause; import org.apache.asterix.lang.common.base.Expression; import org.apache.asterix.lang.common.base.IQueryRewriter; import org.apache.asterix.lang.common.base.IReturningStatement; -import org.apache.asterix.lang.common.clause.LetClause; import org.apache.asterix.lang.common.expression.CallExpr; +import org.apache.asterix.lang.common.expression.ListSliceExpression; import org.apache.asterix.lang.common.expression.VariableExpr; import org.apache.asterix.lang.common.rewrites.LangRewritingContext; import org.apache.asterix.lang.common.statement.FunctionDecl; @@ -57,7 +54,6 @@ import org.apache.asterix.lang.sqlpp.clause.SelectSetOperation; import org.apache.asterix.lang.sqlpp.clause.UnnestClause; import org.apache.asterix.lang.sqlpp.expression.CaseExpression; -import org.apache.asterix.lang.common.expression.ListSliceExpression; import org.apache.asterix.lang.sqlpp.expression.SelectExpression; import org.apache.asterix.lang.sqlpp.expression.WindowExpression; import org.apache.asterix.lang.sqlpp.parser.FunctionParser; @@ -73,19 +69,20 @@ import org.apache.asterix.lang.sqlpp.rewrites.visitor.SqlppGroupByVisitor; import org.apache.asterix.lang.sqlpp.rewrites.visitor.SqlppInlineUdfsVisitor; import org.apache.asterix.lang.sqlpp.rewrites.visitor.SqlppListInputFunctionRewriteVisitor; -import org.apache.asterix.lang.sqlpp.rewrites.visitor.SqlppWindowRewriteVisitor; import org.apache.asterix.lang.sqlpp.rewrites.visitor.SqlppWindowAggregationSugarVisitor; +import org.apache.asterix.lang.sqlpp.rewrites.visitor.SqlppWindowRewriteVisitor; import org.apache.asterix.lang.sqlpp.rewrites.visitor.SubstituteGroupbyExpressionWithVariableVisitor; import org.apache.asterix.lang.sqlpp.rewrites.visitor.VariableCheckAndRewriteVisitor; import org.apache.asterix.lang.sqlpp.struct.SetOperationRight; import org.apache.asterix.lang.sqlpp.util.FunctionMapUtil; import org.apache.asterix.lang.sqlpp.util.SqlppAstPrintUtil; import org.apache.asterix.lang.sqlpp.util.SqlppVariableUtil; -import org.apache.asterix.lang.sqlpp.visitor.FreeVariableVisitor; -import org.apache.asterix.lang.sqlpp.visitor.base.AbstractSqlppQueryExpressionVisitor; import org.apache.asterix.lang.sqlpp.visitor.base.ISqlppVisitor; import org.apache.asterix.metadata.declared.MetadataProvider; import org.apache.hyracks.algebricks.common.utils.Pair; +import org.apache.hyracks.util.LogRedactionUtil; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; public class SqlppQueryRewriter implements IQueryRewriter { @@ -292,7 +289,7 @@ private void logExpression(String p0, String p1) throws CompilationException { if (isLogEnabled) { - LOGGER.trace("{} {}\n{}", p0, p1, SqlppAstPrintUtil.toString(topExpr)); + LOGGER.trace("{} {}\n{}", p0, p1, LogRedactionUtil.userData(SqlppAstPrintUtil.toString(topExpr))); } } diff --git a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/util/ExpressionToVariableUtil.java b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/util/ExpressionToVariableUtil.java index 4842026..d8534be 100644 --- a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/util/ExpressionToVariableUtil.java +++ b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/util/ExpressionToVariableUtil.java @@ -27,6 +27,7 @@ import org.apache.asterix.lang.common.struct.VarIdentifier; import org.apache.asterix.lang.sqlpp.parser.ParseException; import org.apache.asterix.lang.sqlpp.parser.SqlppParseException; +import org.apache.hyracks.util.LogRedactionUtil; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -54,7 +55,7 @@ throw new SqlppParseException(expr.getSourceLocation(), e.getLocalizedMessage()); } throw new SqlppParseException(expr.getSourceLocation(), - "Need an alias for the enclosed expression:\n" + exprText); + "Need an alias for the enclosed expression:\n" + LogRedactionUtil.userData(exprText)); } } diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/PointableHelper.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/PointableHelper.java index 1f086b8..068f239 100644 --- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/PointableHelper.java +++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/PointableHelper.java @@ -32,6 +32,7 @@ import org.apache.hyracks.data.std.api.IValueReference; import org.apache.hyracks.data.std.primitive.UTF8StringPointable; import org.apache.hyracks.data.std.primitive.VoidPointable; +import org.apache.hyracks.util.LogRedactionUtil; import org.apache.hyracks.util.string.UTF8StringWriter; /** @@ -134,7 +135,7 @@ } utf8Writer.writeUTF8(str, output); } catch (IOException e) { - throw new HyracksDataException("Could not serialize " + str); + throw new HyracksDataException("Could not serialize " + LogRedactionUtil.userData(str)); } } diff --git a/hyracks-fullstack/algebricks/algebricks-core/pom.xml b/hyracks-fullstack/algebricks/algebricks-core/pom.xml index 53ee28b..55aa465 100644 --- a/hyracks-fullstack/algebricks/algebricks-core/pom.xml +++ b/hyracks-fullstack/algebricks/algebricks-core/pom.xml @@ -44,6 +44,11 @@ <dependencies> <dependency> <groupId>org.apache.hyracks</groupId> + <artifactId>hyracks-util</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>org.apache.hyracks</groupId> <artifactId>hyracks-dataflow-std</artifactId> <version>${project.version}</version> </dependency> diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/AbstractLogicalOperator.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/AbstractLogicalOperator.java index d4a9d37..571a30a 100644 --- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/AbstractLogicalOperator.java +++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/AbstractLogicalOperator.java @@ -18,6 +18,8 @@ */ package org.apache.hyracks.algebricks.core.algebra.operators.logical; +import static org.apache.hyracks.api.exceptions.ErrorCode.PHYS_OPERATOR_NOT_SET; + import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -164,7 +166,7 @@ throws AlgebricksException { if (bJobGenEnabled) { if (physicalOperator == null) { - throw new AlgebricksException("Physical operator not set for operator: " + this); + throw AlgebricksException.create(PHYS_OPERATOR_NOT_SET, getSourceLocation(), this.getOperatorTag()); } physicalOperator.contributeRuntimeOperator(builder, context, this, propagatedSchema, inputSchemas, outerPlanSchema); diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/FDsAndEquivClassesVisitor.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/FDsAndEquivClassesVisitor.java index 7042794..f5f5f96 100644 --- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/FDsAndEquivClassesVisitor.java +++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/FDsAndEquivClassesVisitor.java @@ -86,6 +86,7 @@ import org.apache.hyracks.algebricks.core.algebra.properties.LocalGroupingProperty; import org.apache.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor; import org.apache.hyracks.algebricks.core.config.AlgebricksConfig; +import org.apache.hyracks.util.LogRedactionUtil; public class FDsAndEquivClassesVisitor implements ILogicalOperatorVisitor<Void, IOptimizationContext> { @@ -316,9 +317,9 @@ } } if (changed && AlgebricksConfig.ALGEBRICKS_LOGGER.isTraceEnabled()) { - AlgebricksConfig.ALGEBRICKS_LOGGER - .trace(">>>> Group-by list changed from " + GroupByOperator.veListToString(gByList) + " to " - + GroupByOperator.veListToString(newGbyList) + ".\n"); + AlgebricksConfig.ALGEBRICKS_LOGGER.trace(">>>> Group-by list changed from {} to {}.\n", + LogRedactionUtil.userData(GroupByOperator.veListToString(gByList)), + LogRedactionUtil.userData(GroupByOperator.veListToString(newGbyList))); } gByList.clear(); gByList.addAll(newGbyList); diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/AbstractUnnestPOperator.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/AbstractUnnestPOperator.java index 0602258..5d54e1d 100644 --- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/AbstractUnnestPOperator.java +++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/AbstractUnnestPOperator.java @@ -36,6 +36,7 @@ import org.apache.hyracks.algebricks.runtime.base.IUnnestingEvaluatorFactory; import org.apache.hyracks.algebricks.runtime.operators.std.UnnestRuntimeFactory; import org.apache.hyracks.api.dataflow.value.RecordDescriptor; +import org.apache.hyracks.util.LogRedactionUtil; public abstract class AbstractUnnestPOperator extends AbstractScanPOperator { private final boolean leftOuter; @@ -73,7 +74,8 @@ } } if (exit) { - throw new AlgebricksException("Unnest expression " + unnestExpr + " is not an unnesting function call."); + throw new AlgebricksException("Unnest expression " + LogRedactionUtil.userData(unnestExpr.toString()) + + " is not an unnesting function call."); } UnnestingFunctionCallExpression agg = (UnnestingFunctionCallExpression) unnestExpr; IUnnestingEvaluatorFactory unnestingFactory = expressionRuntimeProvider.createUnnestingFunctionFactory(agg, diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/jobgen/impl/JobBuilder.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/jobgen/impl/JobBuilder.java index 526add1..c8fff99 100644 --- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/jobgen/impl/JobBuilder.java +++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/jobgen/impl/JobBuilder.java @@ -18,6 +18,8 @@ */ package org.apache.hyracks.algebricks.core.jobgen.impl; +import static org.apache.hyracks.api.exceptions.ErrorCode.DESCRIPTOR_GENERATION_ERROR; + import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -311,7 +313,7 @@ } Integer metaOpKey = algebraicOpBelongingToMetaAsterixOp.get(op); if (metaOpKey == null) { - throw new AlgebricksException("Could not generate operator descriptor for operator " + op); + throw AlgebricksException.create(DESCRIPTOR_GENERATION_ERROR, op.getSourceLocation(), op.getOperatorTag()); } return metaAsterixOps.get(metaOpKey); } diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/rewriter/base/AbstractRuleController.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/rewriter/base/AbstractRuleController.java index 370ec6d..08fe460 100644 --- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/rewriter/base/AbstractRuleController.java +++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/rewriter/base/AbstractRuleController.java @@ -31,6 +31,7 @@ import org.apache.hyracks.algebricks.core.algebra.prettyprint.LogicalOperatorPrettyPrintVisitor; import org.apache.hyracks.algebricks.core.algebra.prettyprint.PlanPrettyPrinter; import org.apache.hyracks.algebricks.core.config.AlgebricksConfig; +import org.apache.hyracks.util.LogRedactionUtil; public abstract class AbstractRuleController { @@ -47,7 +48,7 @@ * Each rewriting strategy may differ in the * * @param root - * @param ruleClasses + * @param rules * @return true iff one of the rules in the collection fired */ public abstract boolean rewriteWithRuleCollection(Mutable<ILogicalOperator> root, @@ -74,12 +75,11 @@ return null; } - private void printRuleApplication(IAlgebraicRewriteRule rule, String beforePlan, String afterPlan) - throws AlgebricksException { + private void printRuleApplication(IAlgebraicRewriteRule rule, String beforePlan, String afterPlan) { if (AlgebricksConfig.ALGEBRICKS_LOGGER.isTraceEnabled()) { - AlgebricksConfig.ALGEBRICKS_LOGGER.trace(">>>> Rule " + rule.getClass() + " fired.\n"); - AlgebricksConfig.ALGEBRICKS_LOGGER.trace(">>>> Before plan\n" + beforePlan + "\n"); - AlgebricksConfig.ALGEBRICKS_LOGGER.trace(">>>> After plan\n" + afterPlan + "\n"); + AlgebricksConfig.ALGEBRICKS_LOGGER.trace(">> Rule " + rule.getClass() + " fired.\n"); + AlgebricksConfig.ALGEBRICKS_LOGGER.trace(">> Before plan\n" + LogRedactionUtil.userData(beforePlan) + "\n"); + AlgebricksConfig.ALGEBRICKS_LOGGER.trace(">> After plan\n" + LogRedactionUtil.userData(afterPlan) + "\n"); } } diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/rewriter/base/HeuristicOptimizer.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/rewriter/base/HeuristicOptimizer.java index 2aacc0e..66ee453 100644 --- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/rewriter/base/HeuristicOptimizer.java +++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/rewriter/base/HeuristicOptimizer.java @@ -33,6 +33,7 @@ import org.apache.hyracks.algebricks.core.algebra.prettyprint.LogicalOperatorPrettyPrintVisitor; import org.apache.hyracks.algebricks.core.algebra.prettyprint.PlanPrettyPrinter; import org.apache.hyracks.algebricks.core.config.AlgebricksConfig; +import org.apache.hyracks.util.LogRedactionUtil; import org.apache.logging.log4j.Level; public class HeuristicOptimizer { @@ -87,7 +88,8 @@ final LogicalOperatorPrettyPrintVisitor pvisitor = context.getPrettyPrintVisitor(); pvisitor.reset(new AlgebricksAppendable()); PlanPrettyPrinter.printPlan(plan, pvisitor, 0); - AlgebricksConfig.ALGEBRICKS_LOGGER.log(lvl, name + ":\n" + pvisitor.get().toString()); + AlgebricksConfig.ALGEBRICKS_LOGGER.log(lvl, + name + ":\n" + LogRedactionUtil.userData(pvisitor.get().toString())); } } diff --git a/hyracks-fullstack/algebricks/algebricks-rewriter/pom.xml b/hyracks-fullstack/algebricks/algebricks-rewriter/pom.xml index f10ca9a..af48b78 100644 --- a/hyracks-fullstack/algebricks/algebricks-rewriter/pom.xml +++ b/hyracks-fullstack/algebricks/algebricks-rewriter/pom.xml @@ -39,6 +39,11 @@ <dependencies> <dependency> <groupId>org.apache.hyracks</groupId> + <artifactId>hyracks-util</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>org.apache.hyracks</groupId> <artifactId>algebricks-core</artifactId> <version>${project.version}</version> </dependency> diff --git a/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/EnforceStructuralPropertiesRule.java b/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/EnforceStructuralPropertiesRule.java index 96e2e53..de9b4bc 100644 --- a/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/EnforceStructuralPropertiesRule.java +++ b/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/EnforceStructuralPropertiesRule.java @@ -106,6 +106,7 @@ import org.apache.hyracks.algebricks.rewriter.util.PhysicalOptimizationsUtil; import org.apache.hyracks.api.exceptions.SourceLocation; import org.apache.hyracks.dataflow.common.data.partition.range.RangeMap; +import org.apache.hyracks.util.LogRedactionUtil; public class EnforceStructuralPropertiesRule implements IAlgebraicRewriteRule { @@ -901,7 +902,7 @@ LogicalOperatorPrettyPrintVisitor pvisitor = new LogicalOperatorPrettyPrintVisitor(); PlanPrettyPrinter.printOperator(op, pvisitor, 0); if (AlgebricksConfig.ALGEBRICKS_LOGGER.isTraceEnabled()) { - AlgebricksConfig.ALGEBRICKS_LOGGER.trace(pvisitor.get().toString()); + AlgebricksConfig.ALGEBRICKS_LOGGER.trace(LogRedactionUtil.userData(pvisitor.get().toString())); } } diff --git a/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/SetAlgebricksPhysicalOperatorsRule.java b/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/SetAlgebricksPhysicalOperatorsRule.java index 4273553..612f79e 100644 --- a/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/SetAlgebricksPhysicalOperatorsRule.java +++ b/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/SetAlgebricksPhysicalOperatorsRule.java @@ -18,6 +18,8 @@ */ package org.apache.hyracks.algebricks.rewriter.rules; +import static org.apache.hyracks.api.exceptions.ErrorCode.ORDER_EXPR_NOT_NORMALIZED; + import java.util.ArrayList; import java.util.List; @@ -222,7 +224,7 @@ for (Pair<IOrder, Mutable<ILogicalExpression>> p : oo.getOrderExpressions()) { ILogicalExpression e = p.second.getValue(); if (e.getExpressionTag() != LogicalExpressionTag.VARIABLE) { - throw new AlgebricksException("Order expression " + e + " has not been normalized."); + throw AlgebricksException.create(ORDER_EXPR_NOT_NORMALIZED, e.getSourceLocation()); } } if (topLevelOp) { diff --git a/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/util/JoinUtils.java b/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/util/JoinUtils.java index 6306338..3042548 100644 --- a/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/util/JoinUtils.java +++ b/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/util/JoinUtils.java @@ -116,8 +116,8 @@ ILogicalPropertiesVector v = context.getLogicalPropertiesVector(opBuild); boolean loggerTraceEnabled = AlgebricksConfig.ALGEBRICKS_LOGGER.isTraceEnabled(); if (loggerTraceEnabled) { - AlgebricksConfig.ALGEBRICKS_LOGGER - .trace("// HybridHashJoin inner branch -- Logical properties for " + opBuild + ": " + v + "\n"); + AlgebricksConfig.ALGEBRICKS_LOGGER.trace("// HybridHashJoin inner branch -- Logical properties for " + + opBuild.getOperatorTag() + ": " + v + "\n"); } if (v != null) { int size2 = v.getMaxOutputFrames(); @@ -125,7 +125,7 @@ if (size2 > 0 && size2 * hhj.getFudgeFactor() <= hhj.getMemSizeInFrames()) { if (loggerTraceEnabled) { AlgebricksConfig.ALGEBRICKS_LOGGER - .trace("// HybridHashJoin inner branch " + opBuild + " fits in memory\n"); + .trace("// HybridHashJoin inner branch " + opBuild.getOperatorTag() + " fits in memory\n"); } // maintains the local properties on the probe side op.setPhysicalOperator( diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/ErrorCode.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/ErrorCode.java index 690870c..d6860f7 100644 --- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/ErrorCode.java +++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/ErrorCode.java @@ -157,6 +157,9 @@ // Compilation error codes. public static final int RULECOLLECTION_NOT_INSTANCE_OF_LIST = 10000; public static final int CANNOT_COMPOSE_PART_CONSTRAINTS = 10001; + public static final int PHYS_OPERATOR_NOT_SET = 10002; + public static final int DESCRIPTOR_GENERATION_ERROR = 10003; + public static final int ORDER_EXPR_NOT_NORMALIZED = 10004; private static class Holder { private static final Map<Integer, String> errorMessageMap; diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/resources/errormsg/en.properties b/hyracks-fullstack/hyracks/hyracks-api/src/main/resources/errormsg/en.properties index 413bbee..e95bf76 100644 --- a/hyracks-fullstack/hyracks/hyracks-api/src/main/resources/errormsg/en.properties +++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/resources/errormsg/en.properties @@ -139,3 +139,6 @@ 10000 = The given rule collection %1$s is not an instance of the List class. 10001 = Cannot compose partition constraint %1$s with %2$s +10002 = Physical operator not set for operator: %1$s +10003 = Could not generate operator descriptor for operator %1$s +10004 = Order expression has not been normalized diff --git a/hyracks-fullstack/hyracks/hyracks-dataflow-common/src/main/java/org/apache/hyracks/dataflow/common/comm/io/FrameDeserializer.java b/hyracks-fullstack/hyracks/hyracks-dataflow-common/src/main/java/org/apache/hyracks/dataflow/common/comm/io/FrameDeserializer.java index 5a39523..c4fd289 100644 --- a/hyracks-fullstack/hyracks/hyracks-dataflow-common/src/main/java/org/apache/hyracks/dataflow/common/comm/io/FrameDeserializer.java +++ b/hyracks-fullstack/hyracks/hyracks-dataflow-common/src/main/java/org/apache/hyracks/dataflow/common/comm/io/FrameDeserializer.java @@ -26,6 +26,7 @@ import org.apache.hyracks.api.dataflow.value.RecordDescriptor; import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.dataflow.common.comm.util.ByteBufferInputStream; +import org.apache.hyracks.util.LogRedactionUtil; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -71,7 +72,7 @@ for (int i = 0; i < record.length; ++i) { Object instance = recordDescriptor.getFields()[i].deserialize(di); if (LOGGER.isTraceEnabled()) { - LOGGER.trace(i + " " + instance); + LOGGER.trace(i + " " + LogRedactionUtil.userData(instance.toString())); } record[i] = instance; if (FrameConstants.DEBUG_FRAME_IO) { diff --git a/hyracks-fullstack/hyracks/hyracks-dataflow-common/src/main/java/org/apache/hyracks/dataflow/common/comm/io/SerializingDataWriter.java b/hyracks-fullstack/hyracks/hyracks-dataflow-common/src/main/java/org/apache/hyracks/dataflow/common/comm/io/SerializingDataWriter.java index f7b5e3b..ce1d595 100644 --- a/hyracks-fullstack/hyracks/hyracks-dataflow-common/src/main/java/org/apache/hyracks/dataflow/common/comm/io/SerializingDataWriter.java +++ b/hyracks-fullstack/hyracks/hyracks-dataflow-common/src/main/java/org/apache/hyracks/dataflow/common/comm/io/SerializingDataWriter.java @@ -25,6 +25,7 @@ import org.apache.hyracks.api.dataflow.value.RecordDescriptor; import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.dataflow.common.comm.util.FrameUtils; +import org.apache.hyracks.util.LogRedactionUtil; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -82,7 +83,7 @@ for (int i = 0; i < data.length; ++i) { Object instance = data[i]; if (LOGGER.isTraceEnabled()) { - LOGGER.trace(i + " " + instance); + LOGGER.trace(i + " " + LogRedactionUtil.userData(instance.toString())); } tb.addField(recordDescriptor.getFields()[i], instance); } diff --git a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/ILogRedactor.java b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/ILogRedactor.java new file mode 100644 index 0000000..18e9a34 --- /dev/null +++ b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/ILogRedactor.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.hyracks.util; + +public interface ILogRedactor { + + /** + * Redacts user data argument. + * @param text user data to redact. + * @return redacted user data. + */ + String userData(String text); +} diff --git a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/LogRedactionUtil.java b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/LogRedactionUtil.java new file mode 100644 index 0000000..29242be --- /dev/null +++ b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/LogRedactionUtil.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hyracks.util; + +public class LogRedactionUtil { + + private static final ILogRedactor defaultLogRedactor = text -> text; + private static ILogRedactor redactor = defaultLogRedactor; + + private LogRedactionUtil() { + } + + public static void setRedactor(ILogRedactor redactor) { + LogRedactionUtil.redactor = redactor; + } + + public static String userData(String text) { + return redactor.userData(text); + } +} -- To view, visit https://asterix-gerrit.ics.uci.edu/3105 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-MessageType: merged Gerrit-Change-Id: I602c833ba2a055da8fbe8782ec62be683ff4581b Gerrit-PatchSet: 10 Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Owner: Ali Alsuliman <[email protected]> Gerrit-Reviewer: Ali Alsuliman <[email protected]> Gerrit-Reviewer: Anon. E. Moose #1000171 Gerrit-Reviewer: Dmitry Lychagin <[email protected]> Gerrit-Reviewer: Jenkins <[email protected]> Gerrit-Reviewer: Murtadha Hubail <[email protected]>
