DRILL-6137: Fixed join error when one file partition is empty
Project: http://git-wip-us.apache.org/repos/asf/drill/repo Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/2637d65c Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/2637d65c Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/2637d65c Branch: refs/heads/master Commit: 2637d65cdb7291f39b99376ce1d9d4907b84a7ff Parents: cd6ddc9 Author: Timothy Farkas <[email protected]> Authored: Mon Feb 5 12:23:31 2018 -0800 Committer: Ben-Zvi <[email protected]> Committed: Mon Feb 5 19:54:50 2018 -0800 ---------------------------------------------------------------------- .../exec/physical/impl/join/HashJoinBatch.java | 4 +- .../exec/physical/impl/join/HashJoinProbe.java | 2 +- .../impl/join/HashJoinProbeTemplate.java | 10 +++- .../exec/record/AbstractSingleRecordBatch.java | 1 + .../impl/join/TestHashJoinAdvanced.java | 23 +++++++++ .../resources/join/empty_part/part/0_0_0.json | 41 ++++++++++++++++ .../join/empty_part/partsupp/0_0_0.json | 49 ++++++++++++++++++++ .../join/empty_part/partsupp/0_0_1.json | 0 8 files changed, 125 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/drill/blob/2637d65c/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/HashJoinBatch.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/HashJoinBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/HashJoinBatch.java index e087bc8..b126255 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/HashJoinBatch.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/HashJoinBatch.java @@ -205,10 +205,8 @@ public class HashJoinBatch extends AbstractBinaryRecordBatch<HashJoinPOP> { if (state == BatchState.FIRST) { // Build the hash table, using the build side record batches. executeBuildPhase(); - // IterOutcome next = next(HashJoinHelper.LEFT_INPUT, left); hashJoinProbe.setupHashJoinProbe(context, hyperContainer, left, left.getRecordCount(), this, hashTable, - hjHelper, joinType); - + hjHelper, joinType, leftUpstream); // Update the hash table related stats for the operator updateStats(this.hashTable); } http://git-wip-us.apache.org/repos/asf/drill/blob/2637d65c/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/HashJoinProbe.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/HashJoinProbe.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/HashJoinProbe.java index 4ef28e6..c54747e 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/HashJoinProbe.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/HashJoinProbe.java @@ -46,7 +46,7 @@ public interface HashJoinProbe { void setupHashJoinProbe(FragmentContext context, VectorContainer buildBatch, RecordBatch probeBatch, int probeRecordCount, HashJoinBatch outgoing, HashTable hashTable, HashJoinHelper hjHelper, - JoinRelType joinRelType); + JoinRelType joinRelType, RecordBatch.IterOutcome leftStartState); void doSetup(FragmentContext context, VectorContainer buildBatch, RecordBatch probeBatch, RecordBatch outgoing); int probeAndProject() throws SchemaChangeException, ClassTransformationException, IOException; void projectBuildRecord(int buildIndex, int outIndex); http://git-wip-us.apache.org/repos/asf/drill/blob/2637d65c/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/HashJoinProbeTemplate.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/HashJoinProbeTemplate.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/HashJoinProbeTemplate.java index 5c6371a..3d4812b 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/HashJoinProbeTemplate.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/HashJoinProbeTemplate.java @@ -83,7 +83,7 @@ public abstract class HashJoinProbeTemplate implements HashJoinProbe { @Override public void setupHashJoinProbe(FragmentContext context, VectorContainer buildBatch, RecordBatch probeBatch, int probeRecordCount, HashJoinBatch outgoing, HashTable hashTable, - HashJoinHelper hjHelper, JoinRelType joinRelType) { + HashJoinHelper hjHelper, JoinRelType joinRelType, IterOutcome leftStartState) { this.probeBatch = probeBatch; this.probeSchema = probeBatch.getSchema(); @@ -94,6 +94,14 @@ public abstract class HashJoinProbeTemplate implements HashJoinProbe { this.hjHelper = hjHelper; this.outgoingJoinBatch = outgoing; + if (leftStartState == IterOutcome.NONE) { + if (joinRelType == JoinRelType.RIGHT) { + probeState = ProbeState.PROJECT_RIGHT; + } else { + probeState = ProbeState.DONE; + } + } + doSetup(context, buildBatch, probeBatch, outgoing); } http://git-wip-us.apache.org/repos/asf/drill/blob/2637d65c/exec/java-exec/src/main/java/org/apache/drill/exec/record/AbstractSingleRecordBatch.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/record/AbstractSingleRecordBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/record/AbstractSingleRecordBatch.java index 07312a3..4cae92d 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/record/AbstractSingleRecordBatch.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/record/AbstractSingleRecordBatch.java @@ -148,6 +148,7 @@ public abstract class AbstractSingleRecordBatch<T extends PhysicalOperator> exte */ protected IterOutcome handleNullInput() { container.buildSchema(SelectionVectorMode.NONE); + container.setRecordCount(0); return IterOutcome.NONE; } http://git-wip-us.apache.org/repos/asf/drill/blob/2637d65c/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestHashJoinAdvanced.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestHashJoinAdvanced.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestHashJoinAdvanced.java index 7219f0e..b9b97c1 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestHashJoinAdvanced.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestHashJoinAdvanced.java @@ -21,6 +21,8 @@ package org.apache.drill.exec.physical.impl.join; import org.apache.drill.categories.OperatorTest; import org.apache.drill.categories.UnlikelyTest; +import org.apache.drill.exec.ExecConstants; +import org.apache.drill.test.BaseTestQuery; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; @@ -28,6 +30,8 @@ import org.junit.experimental.categories.Category; import java.io.BufferedWriter; import java.io.File; import java.io.FileWriter; +import java.nio.file.Paths; +import java.util.concurrent.ExecutorService; @Category(OperatorTest.class) @@ -36,6 +40,7 @@ public class TestHashJoinAdvanced extends JoinTestBase { // Have to disable merge join, if this testcase is to test "HASH-JOIN". @BeforeClass public static void disableMergeJoin() throws Exception { + dirTestWatcher.copyResourceToRoot(Paths.get("join", "empty_part")); test(DISABLE_MJ); } @@ -174,4 +179,22 @@ public class TestHashJoinAdvanced extends JoinTestBase { public void testHashRightJoinWithEmptyTable() throws Exception { testJoinWithEmptyFile(dirTestWatcher.getRootDir(), "right outer", new String[] {HJ_PATTERN, RIGHT_JOIN_TYPE}, 0L); } + + @Test // Test for DRILL-6137 fix + public void emptyPartTest() throws Exception { + BaseTestQuery.setSessionOption(ExecConstants.SLICE_TARGET, 1L); + + try { + testBuilder().sqlQuery("select t.p_partkey, t1.ps_suppkey from " + + "dfs.`join/empty_part/part` as t RIGHT JOIN dfs.`join/empty_part/partsupp` as t1 ON t.p_partkey = t1.ps_partkey where t1.ps_partkey > 1").unOrdered() + .baselineColumns("ps_suppkey", "p_partkey") + .baselineValues(3L, 2L) + .baselineValues(2503L, 2L) + .baselineValues(5003L, 2L) + .baselineValues(7503L, 2L) + .go(); + } finally { + BaseTestQuery.resetSessionOption(ExecConstants.SLICE_TARGET); + } + } } http://git-wip-us.apache.org/repos/asf/drill/blob/2637d65c/exec/java-exec/src/test/resources/join/empty_part/part/0_0_0.json ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/join/empty_part/part/0_0_0.json b/exec/java-exec/src/test/resources/join/empty_part/part/0_0_0.json new file mode 100644 index 0000000..0f4d85c --- /dev/null +++ b/exec/java-exec/src/test/resources/join/empty_part/part/0_0_0.json @@ -0,0 +1,41 @@ +{ + "p_partkey" : 1, + "p_name" : "goldenrod lace spring peru powder", + "p_mfgr" : "Manufacturer#1", + "p_brand" : "Brand#13", + "p_type" : "PROMO BURNISHED COPPER", + "p_size" : 7, + "p_container" : "JUMBO PKG", + "p_retailprice" : 901.0, + "p_comment" : "ly. slyly ironi" +} { + "p_partkey" : 2, + "p_name" : "blush rosy metallic lemon navajo", + "p_mfgr" : "Manufacturer#1", + "p_brand" : "Brand#13", + "p_type" : "LARGE BRUSHED BRASS", + "p_size" : 1, + "p_container" : "LG CASE", + "p_retailprice" : 902.0, + "p_comment" : "lar accounts amo" +} { + "p_partkey" : 3, + "p_name" : "dark green antique puff wheat", + "p_mfgr" : "Manufacturer#4", + "p_brand" : "Brand#42", + "p_type" : "STANDARD POLISHED BRASS", + "p_size" : 21, + "p_container" : "WRAP CASE", + "p_retailprice" : 903.0, + "p_comment" : "egular deposits hag" +} { + "p_partkey" : 4, + "p_name" : "chocolate metallic smoke ghost drab", + "p_mfgr" : "Manufacturer#3", + "p_brand" : "Brand#34", + "p_type" : "SMALL PLATED BRASS", + "p_size" : 14, + "p_container" : "MED DRUM", + "p_retailprice" : 904.0, + "p_comment" : "p furiously r" +} http://git-wip-us.apache.org/repos/asf/drill/blob/2637d65c/exec/java-exec/src/test/resources/join/empty_part/partsupp/0_0_0.json ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/join/empty_part/partsupp/0_0_0.json b/exec/java-exec/src/test/resources/join/empty_part/partsupp/0_0_0.json new file mode 100644 index 0000000..f0e6834 --- /dev/null +++ b/exec/java-exec/src/test/resources/join/empty_part/partsupp/0_0_0.json @@ -0,0 +1,49 @@ +{ + "ps_partkey" : 1, + "ps_suppkey" : 2, + "ps_availqty" : 3325, + "ps_supplycost" : 771.64, + "ps_comment" : ", even theodolites. regular, final theodolites eat after the carefully pending foxes. furiously regular deposits sleep slyly. carefully bold realms above the ironic dependencies haggle careful" +} { + "ps_partkey" : 1, + "ps_suppkey" : 2502, + "ps_availqty" : 8076, + "ps_supplycost" : 993.49, + "ps_comment" : "ven ideas. quickly even packages print. pending multipliers must have to are fluff" +} { + "ps_partkey" : 1, + "ps_suppkey" : 5002, + "ps_availqty" : 3956, + "ps_supplycost" : 337.09, + "ps_comment" : "after the fluffily ironic deposits? blithely special dependencies integrate furiously even excuses. blithely silent theodolites could have to haggle pending, express requests; fu" +} { + "ps_partkey" : 1, + "ps_suppkey" : 7502, + "ps_availqty" : 4069, + "ps_supplycost" : 357.84, + "ps_comment" : "al, regular dependencies serve carefully after the quickly final pinto beans. furiously even deposits sleep quickly final, silent pinto beans. fluffily reg" +} { + "ps_partkey" : 2, + "ps_suppkey" : 3, + "ps_availqty" : 8895, + "ps_supplycost" : 378.49, + "ps_comment" : "nic accounts. final accounts sleep furiously about the ironic, bold packages. regular, regular accounts" +} { + "ps_partkey" : 2, + "ps_suppkey" : 2503, + "ps_availqty" : 4969, + "ps_supplycost" : 915.27, + "ps_comment" : "ptotes. quickly pending dependencies integrate furiously. fluffily ironic ideas impress blithely above the express accounts. furiously even epitaphs need to wak" +} { + "ps_partkey" : 2, + "ps_suppkey" : 5003, + "ps_availqty" : 8539, + "ps_supplycost" : 438.37, + "ps_comment" : "blithely bold ideas. furiously stealthy packages sleep fluffily. slyly special deposits snooze furiously carefully regular accounts. regular deposits according to the accounts nag carefully slyl" +} { + "ps_partkey" : 2, + "ps_suppkey" : 7503, + "ps_availqty" : 3025, + "ps_supplycost" : 306.39, + "ps_comment" : "olites. deposits wake carefully. even, express requests cajole. carefully regular ex" +} http://git-wip-us.apache.org/repos/asf/drill/blob/2637d65c/exec/java-exec/src/test/resources/join/empty_part/partsupp/0_0_1.json ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/join/empty_part/partsupp/0_0_1.json b/exec/java-exec/src/test/resources/join/empty_part/partsupp/0_0_1.json new file mode 100644 index 0000000..e69de29
