DRILL-4384: Fix Profile regressions
Project: http://git-wip-us.apache.org/repos/asf/drill/repo Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/c95b5432 Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/c95b5432 Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/c95b5432 Branch: refs/heads/master Commit: c95b5432301fe487d64a1fc06e765228469fc3a2 Parents: 0842851 Author: Jacques Nadeau <[email protected]> Authored: Sun Feb 21 10:29:09 2016 -0800 Committer: vkorukanti <[email protected]> Committed: Wed Mar 2 14:08:51 2016 -0800 ---------------------------------------------------------------------- .../drill/exec/ops/BufferManagerImpl.java | 13 ++--- .../apache/drill/exec/ops/OperatorStats.java | 45 ++++++++++++++--- .../drill/exec/work/foreman/QueryManager.java | 51 ++++++++++++++------ 3 files changed, 79 insertions(+), 30 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/drill/blob/c95b5432/exec/java-exec/src/main/java/org/apache/drill/exec/ops/BufferManagerImpl.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/BufferManagerImpl.java b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/BufferManagerImpl.java index e82f3f4..aa00b1c 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/BufferManagerImpl.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/BufferManagerImpl.java @@ -22,6 +22,7 @@ import io.netty.buffer.DrillBuf; import org.apache.drill.exec.memory.BufferAllocator; import com.carrotsearch.hppc.LongObjectHashMap; +import com.carrotsearch.hppc.predicates.LongObjectPredicate; public class BufferManagerImpl implements BufferManager { private LongObjectHashMap<DrillBuf> managedBuffers = new LongObjectHashMap<>(); @@ -33,13 +34,13 @@ public class BufferManagerImpl implements BufferManager { @Override public void close() { - final Object[] mbuffers = ((LongObjectHashMap<Object>) (Object) managedBuffers).values; - for (int i = 0; i < mbuffers.length; i++) { - final DrillBuf buf = (DrillBuf) mbuffers[i]; - if (buf != null) { - buf.release(); + managedBuffers.forEach(new LongObjectPredicate<DrillBuf>() { + @Override + public boolean apply(long key, DrillBuf value) { + value.release(); + return true; } - } + }); managedBuffers.clear(); } http://git-wip-us.apache.org/repos/asf/drill/blob/c95b5432/exec/java-exec/src/main/java/org/apache/drill/exec/ops/OperatorStats.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/OperatorStats.java b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/OperatorStats.java index e79eab3..271f734 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/OperatorStats.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/OperatorStats.java @@ -22,12 +22,15 @@ import java.util.Iterator; import org.apache.drill.exec.memory.BufferAllocator; import org.apache.drill.exec.proto.UserBitShared.MetricValue; import org.apache.drill.exec.proto.UserBitShared.OperatorProfile; +import org.apache.drill.exec.proto.UserBitShared.OperatorProfile.Builder; import org.apache.drill.exec.proto.UserBitShared.StreamProfile; import com.carrotsearch.hppc.IntDoubleHashMap; import com.carrotsearch.hppc.IntLongHashMap; import com.carrotsearch.hppc.cursors.IntDoubleCursor; import com.carrotsearch.hppc.cursors.IntLongCursor; +import com.carrotsearch.hppc.procedures.IntDoubleProcedure; +import com.carrotsearch.hppc.procedures.IntLongProcedure; public class OperatorStats { static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(OperatorStats.class); @@ -216,19 +219,45 @@ public class OperatorStats { } } + private class LongProc implements IntLongProcedure { + + private final OperatorProfile.Builder builder; + + public LongProc(Builder builder) { + super(); + this.builder = builder; + } + + @Override + public void apply(int key, long value) { + builder.addMetric(MetricValue.newBuilder().setMetricId(key).setLongValue(value)); + } + + } + public void addLongMetrics(OperatorProfile.Builder builder) { - for (int i = 0; i < longMetrics.keys.length; i++) { - if (longMetrics.keys[i] != 0) { - builder.addMetric(MetricValue.newBuilder().setMetricId(longMetrics.keys[i]).setLongValue(longMetrics.values[i])); - } + if (longMetrics.size() > 0) { + longMetrics.forEach(new LongProc(builder)); } } + private class DoubleProc implements IntDoubleProcedure { + private final OperatorProfile.Builder builder; + + public DoubleProc(Builder builder) { + super(); + this.builder = builder; + } + + @Override + public void apply(int key, double value) { + builder.addMetric(MetricValue.newBuilder().setMetricId(key).setDoubleValue(value)); + } + + } public void addDoubleMetrics(OperatorProfile.Builder builder) { - for (int i = 0; i < longMetrics.keys.length; i++) { - if (doubleMetrics.keys[i] != 0) { - builder.addMetric(MetricValue.newBuilder().setMetricId(doubleMetrics.keys[i]).setDoubleValue(doubleMetrics.values[i])); - } + if (doubleMetrics.size() > 0) { + doubleMetrics.forEach(new DoubleProc(builder)); } } http://git-wip-us.apache.org/repos/asf/drill/blob/c95b5432/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/QueryManager.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/QueryManager.java b/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/QueryManager.java index 39fa5cb..f4ca42b 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/QueryManager.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/QueryManager.java @@ -42,6 +42,7 @@ import org.apache.drill.exec.proto.UserBitShared.MajorFragmentProfile; import org.apache.drill.exec.proto.UserBitShared.QueryId; import org.apache.drill.exec.proto.UserBitShared.QueryInfo; import org.apache.drill.exec.proto.UserBitShared.QueryProfile; +import org.apache.drill.exec.proto.UserBitShared.QueryProfile.Builder; import org.apache.drill.exec.proto.UserBitShared.QueryResult.QueryState; import org.apache.drill.exec.proto.UserProtos.RunQuery; import org.apache.drill.exec.proto.helper.QueryIdHelper; @@ -55,6 +56,7 @@ import org.apache.drill.exec.work.EndpointListener; import org.apache.drill.exec.work.foreman.Foreman.StateListener; import com.carrotsearch.hppc.IntObjectHashMap; +import com.carrotsearch.hppc.predicates.IntObjectPredicate; import com.google.common.base.Preconditions; import com.google.common.collect.Lists; import com.google.common.collect.Maps; @@ -355,26 +357,43 @@ public class QueryManager implements AutoCloseable { profileBuilder.setPlan(planText); } - for (int i = 0; i < fragmentDataMap.keys.length; i++) { - if (fragmentDataMap.keys[i] != 0) { - final int majorFragmentId = fragmentDataMap.keys[i]; - final IntObjectHashMap<FragmentData> minorMap = - (IntObjectHashMap<FragmentData>) ((Object[]) fragmentDataMap.values)[i]; - final MajorFragmentProfile.Builder fb = MajorFragmentProfile.newBuilder() - .setMajorFragmentId(majorFragmentId); - for (int v = 0; v < minorMap.keys.length; v++) { - if (minorMap.keys[v] != 0) { - final FragmentData data = (FragmentData) ((Object[]) minorMap.values)[v]; - fb.addMinorFragmentProfile(data.getProfile()); - } - } - profileBuilder.addFragmentProfile(fb); - } - } + fragmentDataMap.forEach(new OuterIter(profileBuilder)); return profileBuilder.build(); } + private class OuterIter implements IntObjectPredicate<IntObjectHashMap<FragmentData>> { + private final QueryProfile.Builder profileBuilder; + + public OuterIter(Builder profileBuilder) { + this.profileBuilder = profileBuilder; + } + + @Override + public boolean apply(final int majorFragmentId, final IntObjectHashMap<FragmentData> minorMap) { + final MajorFragmentProfile.Builder builder = MajorFragmentProfile.newBuilder().setMajorFragmentId(majorFragmentId); + minorMap.forEach(new InnerIter(builder)); + profileBuilder.addFragmentProfile(builder); + return true; + } + + } + + private class InnerIter implements IntObjectPredicate<FragmentData> { + private final MajorFragmentProfile.Builder builder; + + public InnerIter(MajorFragmentProfile.Builder fb) { + this.builder = fb; + } + + @Override + public boolean apply(int key, FragmentData data) { + builder.addMinorFragmentProfile(data.getProfile()); + return true; + } + + } + void setPlanText(final String planText) { this.planText = planText; }
