This is an automated email from the ASF dual-hosted git repository.
dlych pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new 1783667 [NO ISSUE][COMP] Support runtime warnings during constant
folding
1783667 is described below
commit 17836677fe2e4840625ed420377e439800dbc993
Author: Dmitry Lychagin <[email protected]>
AuthorDate: Thu Jul 11 10:40:08 2019 -0700
[NO ISSUE][COMP] Support runtime warnings during constant folding
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Add IWarningCollector accessor to IEvaluatorContext
- Provide compiler's warning collector to function evaluators
during constant folding, so they can emit warnings
Change-Id: I61dd53adfbcad2f8479333579185258876f074e9
Reviewed-on: https://asterix-gerrit.ics.uci.edu/3484
Contrib: Jenkins <[email protected]>
Integration-Tests: Jenkins <[email protected]>
Tested-by: Jenkins <[email protected]>
Reviewed-by: Dmitry Lychagin <[email protected]>
Reviewed-by: Ali Alsuliman <[email protected]>
---
.../asterix/optimizer/rules/ConstantFoldingRule.java | 15 +++++++++++++--
.../asterix/common/exceptions/WarningCollector.java | 6 ++++++
.../aggregates/std/AbstractMinMaxAggregateFunction.java | 4 ++--
.../algebricks/runtime/base/IEvaluatorContext.java | 6 ++++++
.../algebricks/runtime/evaluators/EvaluatorContext.java | 17 +++++++++++++++++
.../apache/hyracks/api/context/IHyracksTaskContext.java | 9 ++-------
.../main/java/org/apache/hyracks/control/nc/Task.java | 8 ++++++--
.../apache/hyracks/test/support/TestTaskContext.java | 8 +++-----
.../java/org/apache/hyracks/test/support/TestUtils.java | 5 +++++
.../invertedindex/util/LSMInvertedIndexTestUtils.java | 7 ++++---
10 files changed, 64 insertions(+), 21 deletions(-)
diff --git
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ConstantFoldingRule.java
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ConstantFoldingRule.java
index b095fe1..4c6f338 100644
---
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ConstantFoldingRule.java
+++
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ConstantFoldingRule.java
@@ -28,6 +28,7 @@ import java.util.Set;
import org.apache.asterix.common.config.GlobalConfig;
import org.apache.asterix.common.dataflow.ICcApplicationContext;
+import org.apache.asterix.common.exceptions.WarningCollector;
import org.apache.asterix.dataflow.data.common.ExpressionTypeComputer;
import org.apache.asterix.dataflow.data.nontagged.MissingWriterFactory;
import org.apache.asterix.formats.nontagged.ADMPrinterFactoryProvider;
@@ -78,6 +79,7 @@ import
org.apache.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionVis
import org.apache.hyracks.algebricks.core.config.AlgebricksConfig;
import org.apache.hyracks.algebricks.core.jobgen.impl.JobGenContext;
import org.apache.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+import org.apache.hyracks.algebricks.runtime.base.IEvaluatorContext;
import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator;
import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
import org.apache.hyracks.algebricks.runtime.evaluators.EvaluatorContext;
@@ -172,7 +174,7 @@ public class ConstantFoldingRule implements
IAlgebraicRewriteRule {
if (context.checkIfInDontApplySet(this, op)) {
return false;
}
-
+ cfv.reset(context);
return op.acceptExpressionTransform(cfv);
}
@@ -182,6 +184,13 @@ public class ConstantFoldingRule implements
IAlgebraicRewriteRule {
private final IPointable p = VoidPointable.FACTORY.createPointable();
private final ByteBufferInputStream bbis = new ByteBufferInputStream();
private final DataInputStream dis = new DataInputStream(bbis);
+ private final WarningCollector warningCollector = new
WarningCollector();
+ private final IEvaluatorContext evalContext = new
EvaluatorContext(warningCollector);
+ private IOptimizationContext optContext;
+
+ private void reset(IOptimizationContext context) {
+ optContext = context;
+ }
@Override
public boolean transform(Mutable<ILogicalExpression> exprRef) throws
AlgebricksException {
@@ -248,7 +257,8 @@ public class ConstantFoldingRule implements
IAlgebraicRewriteRule {
IScalarEvaluatorFactory fact =
jobGenCtx.getExpressionRuntimeProvider().createEvaluatorFactory(expr,
_emptyTypeEnv, _emptySchemas, jobGenCtx);
- IScalarEvaluator eval = fact.createScalarEvaluator(new
EvaluatorContext(null));
+ warningCollector.clear();
+ IScalarEvaluator eval =
fact.createScalarEvaluator(evalContext);
eval.evaluate(null, p);
IAType returnType = (IAType) _emptyTypeEnv.getType(expr);
ATypeTag runtimeType = PointableHelper.getTypeTag(p);
@@ -262,6 +272,7 @@ public class ConstantFoldingRule implements
IAlgebraicRewriteRule {
jobGenCtx.getSerializerDeserializerProvider().getSerializerDeserializer(returnType);
bbis.setByteBuffer(ByteBuffer.wrap(p.getByteArray(),
p.getStartOffset(), p.getLength()), 0);
IAObject o = (IAObject) serde.deserialize(dis);
+ warningCollector.getWarnings(optContext.getWarningCollector());
return new Pair<>(true, new ConstantExpression(new
AsterixConstantValue(o)));
} catch (HyracksDataException | AlgebricksException e) {
if (AlgebricksConfig.ALGEBRICKS_LOGGER.isTraceEnabled()) {
diff --git
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/WarningCollector.java
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/WarningCollector.java
index 2f0af3b..a624e71 100644
---
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/WarningCollector.java
+++
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/WarningCollector.java
@@ -46,4 +46,10 @@ public final class WarningCollector implements
IWarningCollector {
public void getWarnings(Collection<? super Warning> outWarnings) {
outWarnings.addAll(warnings);
}
+
+ public void getWarnings(IWarningCollector outWarningCollector) {
+ for (Warning warning : warnings) {
+ outWarningCollector.warn(warning);
+ }
+ }
}
diff --git
a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/aggregates/std/AbstractMinMaxAggregateFunction.java
b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/aggregates/std/AbstractMinMaxAggregateFunction.java
index b653135..aa45922 100644
---
a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/aggregates/std/AbstractMinMaxAggregateFunction.java
+++
b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/aggregates/std/AbstractMinMaxAggregateFunction.java
@@ -156,13 +156,13 @@ public abstract class AbstractMinMaxAggregateFunction
extends AbstractAggregateF
}
private void handleIncompatibleInput(ATypeTag typeTag) {
- context.getTaskContext()
+ context.getWarningCollector()
.warn(WarningUtil.forAsterix(sourceLoc,
ErrorCode.TYPE_INCOMPATIBLE, "min/max", aggType, typeTag));
this.aggType = ATypeTag.NULL;
}
private void handleUnsupportedInput(ATypeTag typeTag) {
- context.getTaskContext()
+ context.getWarningCollector()
.warn(WarningUtil.forAsterix(sourceLoc,
ErrorCode.TYPE_UNSUPPORTED, "min/max", typeTag));
this.aggType = ATypeTag.NULL;
}
diff --git
a/hyracks-fullstack/algebricks/algebricks-runtime/src/main/java/org/apache/hyracks/algebricks/runtime/base/IEvaluatorContext.java
b/hyracks-fullstack/algebricks/algebricks-runtime/src/main/java/org/apache/hyracks/algebricks/runtime/base/IEvaluatorContext.java
index efa613d..1c14c1d 100644
---
a/hyracks-fullstack/algebricks/algebricks-runtime/src/main/java/org/apache/hyracks/algebricks/runtime/base/IEvaluatorContext.java
+++
b/hyracks-fullstack/algebricks/algebricks-runtime/src/main/java/org/apache/hyracks/algebricks/runtime/base/IEvaluatorContext.java
@@ -20,6 +20,7 @@
package org.apache.hyracks.algebricks.runtime.base;
import org.apache.hyracks.api.context.IHyracksTaskContext;
+import org.apache.hyracks.api.exceptions.IWarningCollector;
/**
* Context for runtime function evaluators
@@ -30,4 +31,9 @@ public interface IEvaluatorContext {
* is being executed by the constant folding rule at compile time.
*/
IHyracksTaskContext getTaskContext();
+
+ /**
+ * Returns a warning collector, never {@code null}
+ */
+ IWarningCollector getWarningCollector();
}
diff --git
a/hyracks-fullstack/algebricks/algebricks-runtime/src/main/java/org/apache/hyracks/algebricks/runtime/evaluators/EvaluatorContext.java
b/hyracks-fullstack/algebricks/algebricks-runtime/src/main/java/org/apache/hyracks/algebricks/runtime/evaluators/EvaluatorContext.java
index 2113e57..beecee9 100644
---
a/hyracks-fullstack/algebricks/algebricks-runtime/src/main/java/org/apache/hyracks/algebricks/runtime/evaluators/EvaluatorContext.java
+++
b/hyracks-fullstack/algebricks/algebricks-runtime/src/main/java/org/apache/hyracks/algebricks/runtime/evaluators/EvaluatorContext.java
@@ -19,19 +19,36 @@
package org.apache.hyracks.algebricks.runtime.evaluators;
+import java.util.Objects;
+
import org.apache.hyracks.algebricks.runtime.base.IEvaluatorContext;
import org.apache.hyracks.api.context.IHyracksTaskContext;
+import org.apache.hyracks.api.exceptions.IWarningCollector;
public final class EvaluatorContext implements IEvaluatorContext {
private final IHyracksTaskContext taskContext;
+ private final IWarningCollector warningCollector;
+
public EvaluatorContext(IHyracksTaskContext taskContext) {
this.taskContext = taskContext;
+ this.warningCollector = taskContext.getWarningCollector();
+ }
+
+ public EvaluatorContext(IWarningCollector warningCollector) {
+ Objects.requireNonNull(warningCollector);
+ this.taskContext = null;
+ this.warningCollector = warningCollector;
}
@Override
public IHyracksTaskContext getTaskContext() {
return taskContext;
}
+
+ @Override
+ public IWarningCollector getWarningCollector() {
+ return warningCollector;
+ }
}
diff --git
a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/context/IHyracksTaskContext.java
b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/context/IHyracksTaskContext.java
index abe901a..1ffcf9f 100644
---
a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/context/IHyracksTaskContext.java
+++
b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/context/IHyracksTaskContext.java
@@ -25,7 +25,7 @@ import java.util.concurrent.ExecutorService;
import org.apache.hyracks.api.dataflow.TaskAttemptId;
import org.apache.hyracks.api.deployment.DeploymentId;
import org.apache.hyracks.api.exceptions.HyracksException;
-import org.apache.hyracks.api.exceptions.Warning;
+import org.apache.hyracks.api.exceptions.IWarningCollector;
import org.apache.hyracks.api.io.IWorkspaceFileFactory;
import org.apache.hyracks.api.job.IOperatorEnvironment;
import org.apache.hyracks.api.job.JobFlag;
@@ -60,10 +60,5 @@ public interface IHyracksTaskContext
IStatsCollector getStatsCollector();
- /**
- * Adds a warning to this {@link IHyracksTaskContext}
- *
- * @param warning
- */
- void warn(Warning warning);
+ IWarningCollector getWarningCollector();
}
diff --git
a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/Task.java
b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/Task.java
index b8c1496..8dd94f3 100644
---
a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/Task.java
+++
b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/Task.java
@@ -47,6 +47,7 @@ import org.apache.hyracks.api.dataflow.state.IStateObject;
import org.apache.hyracks.api.deployment.DeploymentId;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.exceptions.HyracksException;
+import org.apache.hyracks.api.exceptions.IWarningCollector;
import org.apache.hyracks.api.exceptions.Warning;
import org.apache.hyracks.api.io.FileReference;
import org.apache.hyracks.api.io.IIOManager;
@@ -119,6 +120,8 @@ public class Task implements IHyracksTaskContext,
ICounterContext, Runnable {
private final Set<Warning> warnings;
+ private final IWarningCollector warningCollector;
+
public Task(Joblet joblet, Set<JobFlag> jobFlags, TaskAttemptId taskId,
String displayName,
ExecutorService executor, NodeControllerService ncs,
List<List<PartitionChannel>> inputChannelsFromConnectors) {
@@ -138,6 +141,7 @@ public class Task implements IHyracksTaskContext,
ICounterContext, Runnable {
this.inputChannelsFromConnectors = inputChannelsFromConnectors;
statsCollector = new StatsCollector();
warnings = ConcurrentHashMap.newKeySet();
+ warningCollector = warnings::add;
}
public void setTaskRuntime(IPartitionCollector[] collectors,
IOperatorNodePushable operator) {
@@ -474,8 +478,8 @@ public class Task implements IHyracksTaskContext,
ICounterContext, Runnable {
}
@Override
- public void warn(Warning warning) {
- warnings.add(warning);
+ public IWarningCollector getWarningCollector() {
+ return warningCollector;
}
public boolean isCompleted() {
diff --git
a/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/test/support/TestTaskContext.java
b/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/test/support/TestTaskContext.java
index e328e3d..f264a4b 100644
---
a/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/test/support/TestTaskContext.java
+++
b/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/test/support/TestTaskContext.java
@@ -22,7 +22,6 @@ import java.io.Serializable;
import java.nio.ByteBuffer;
import java.util.EnumSet;
import java.util.HashMap;
-import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ExecutorService;
@@ -33,7 +32,7 @@ import org.apache.hyracks.api.dataflow.TaskAttemptId;
import org.apache.hyracks.api.dataflow.state.IStateObject;
import org.apache.hyracks.api.deployment.DeploymentId;
import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.api.exceptions.Warning;
+import org.apache.hyracks.api.exceptions.IWarningCollector;
import org.apache.hyracks.api.io.FileReference;
import org.apache.hyracks.api.io.IIOManager;
import org.apache.hyracks.api.job.JobFlag;
@@ -51,7 +50,6 @@ public class TestTaskContext implements IHyracksTaskContext {
private Map<Object, IStateObject> stateObjectMap = new HashMap<>();
private Object sharedObject;
private final IStatsCollector statsCollector = new StatsCollector();
- private final Set<Warning> warnings = new HashSet<>();
public TestTaskContext(TestJobletContext jobletContext, TaskAttemptId
taskId) {
this.jobletContext = jobletContext;
@@ -180,7 +178,7 @@ public class TestTaskContext implements IHyracksTaskContext
{
}
@Override
- public void warn(Warning warning) {
- warnings.add(warning);
+ public IWarningCollector getWarningCollector() {
+ return TestUtils.NOOP_WARNING_COLLECTOR;
}
}
diff --git
a/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/test/support/TestUtils.java
b/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/test/support/TestUtils.java
index e36b655..420f7e6 100644
---
a/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/test/support/TestUtils.java
+++
b/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/test/support/TestUtils.java
@@ -37,6 +37,7 @@ import org.apache.hyracks.api.dataflow.OperatorDescriptorId;
import org.apache.hyracks.api.dataflow.TaskAttemptId;
import org.apache.hyracks.api.dataflow.TaskId;
import org.apache.hyracks.api.exceptions.HyracksException;
+import org.apache.hyracks.api.exceptions.IWarningCollector;
import org.apache.hyracks.api.io.IODeviceHandle;
import org.apache.hyracks.api.job.JobId;
import org.apache.hyracks.api.util.CleanupUtils;
@@ -48,6 +49,10 @@ import
org.apache.logging.log4j.core.appender.ConsoleAppender;
import org.apache.logging.log4j.core.config.Configuration;
public class TestUtils {
+
+ public static final IWarningCollector NOOP_WARNING_COLLECTOR = w -> {
+ };
+
public static IHyracksTaskContext create(int frameSize) {
IOManager ioManager = null;
try {
diff --git
a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/org/apache/hyracks/storage/am/lsm/invertedindex/util/LSMInvertedIndexTestUtils.java
b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/org/apache/hyracks/storage/am/lsm/invertedindex/util/LSMInvertedIndexTestUtils.java
index 9021fc5..aec9e35 100644
---
a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/org/apache/hyracks/storage/am/lsm/invertedindex/util/LSMInvertedIndexTestUtils.java
+++
b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/org/apache/hyracks/storage/am/lsm/invertedindex/util/LSMInvertedIndexTestUtils.java
@@ -48,7 +48,7 @@ import org.apache.hyracks.api.deployment.DeploymentId;
import org.apache.hyracks.api.exceptions.ErrorCode;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.exceptions.HyracksException;
-import org.apache.hyracks.api.exceptions.Warning;
+import org.apache.hyracks.api.exceptions.IWarningCollector;
import org.apache.hyracks.api.io.FileReference;
import org.apache.hyracks.api.io.IIOManager;
import org.apache.hyracks.api.job.JobFlag;
@@ -106,6 +106,7 @@ import
org.apache.hyracks.storage.common.IIndexAccessParameters;
import org.apache.hyracks.storage.common.IIndexBulkLoader;
import org.apache.hyracks.storage.common.IIndexCursor;
import org.apache.hyracks.storage.common.MultiComparator;
+import org.apache.hyracks.test.support.TestUtils;
@SuppressWarnings("rawtypes")
public class LSMInvertedIndexTestUtils {
@@ -767,8 +768,8 @@ public class LSMInvertedIndexTestUtils {
}
@Override
- public void warn(Warning warning) {
- // no-op
+ public IWarningCollector getWarningCollector() {
+ return TestUtils.NOOP_WARNING_COLLECTOR;
}
}
