This is an automated email from the ASF dual-hosted git repository.
arina pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
The following commit(s) were added to refs/heads/master by this push:
new 8c37156 DRILL-7413: Test and fix scan operator vectors
8c37156 is described below
commit 8c37156a91ee531e4d1b672a41255cdeb11bdd57
Author: Paul Rogers <[email protected]>
AuthorDate: Sun Oct 20 12:03:28 2019 -0700
DRILL-7413: Test and fix scan operator vectors
Enables vector validation tests for the ScanBatch and all
EasyFormat plugins. Fixes a bug in scan batch that failed to set
the record count in the output container.
Fixes a number of formatting and other issues found while adding
the tests.
---
.../apache/drill/exec/physical/impl/ScanBatch.java | 28 +-
.../physical/impl/validate/BatchValidator.java | 62 +++-
.../apache/drill/exec/record/VectorContainer.java | 5 +
.../exec/store/easy/json/JSONFormatPlugin.java | 24 +-
.../sequencefile/SequenceFileFormatPlugin.java | 9 +-
.../drill/exec/record/vector/TestDateTypes.java | 376 ++++++++++-----------
.../test/resources/parquet/alltypes_repeated.json | 4 +-
7 files changed, 278 insertions(+), 230 deletions(-)
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScanBatch.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScanBatch.java
index cdb36f4..79b40ee 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScanBatch.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScanBatch.java
@@ -17,9 +17,11 @@
*/
package org.apache.drill.exec.physical.impl;
-import
org.apache.drill.shaded.guava.com.google.common.annotations.VisibleForTesting;
-import org.apache.drill.shaded.guava.com.google.common.base.Preconditions;
-import io.netty.buffer.DrillBuf;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
import org.apache.drill.common.exceptions.ExecutionSetupException;
import org.apache.drill.common.exceptions.UserException;
import org.apache.drill.common.expression.SchemaPath;
@@ -54,10 +56,10 @@ import org.apache.drill.exec.vector.AllocationHelper;
import org.apache.drill.exec.vector.NullableVarCharVector;
import org.apache.drill.exec.vector.SchemaChangeCallBack;
import org.apache.drill.exec.vector.ValueVector;
-import java.util.Collections;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
+import
org.apache.drill.shaded.guava.com.google.common.annotations.VisibleForTesting;
+import org.apache.drill.shaded.guava.com.google.common.base.Preconditions;
+
+import io.netty.buffer.DrillBuf;
/**
* Record batch used for a particular scan. Operators against one or more
@@ -155,6 +157,8 @@ public class ScanBatch implements CloseableRecordBatch {
return context;
}
+ public OperatorContext getOperatorContext() { return oContext; }
+
@Override
public BatchSchema getSchema() {
return schema;
@@ -224,7 +228,8 @@ public class ScanBatch implements CloseableRecordBatch {
logger.trace("currentReader.next return recordCount={}", recordCount);
Preconditions.checkArgument(recordCount >= 0, "recordCount from
RecordReader.next() should not be negative");
boolean isNewSchema = mutator.isNewSchema();
- populateImplicitVectorsAndSetCount();
+ populateImplicitVectors();
+ mutator.container.setValueCount(recordCount);
oContext.getStats().batchReceived(0, recordCount, isNewSchema);
boolean toContinueIter = true;
@@ -339,12 +344,8 @@ public class ScanBatch implements CloseableRecordBatch {
}
}
- private void populateImplicitVectorsAndSetCount() {
+ private void populateImplicitVectors() {
mutator.populateImplicitVectors(implicitValues, recordCount);
- for (Map.Entry<String, ValueVector> entry:
mutator.fieldVectorMap().entrySet()) {
- logger.debug("set record count {} for vv {}", recordCount,
entry.getKey());
- entry.getValue().getMutator().setValueCount(recordCount);
- }
}
@Override
@@ -602,6 +603,7 @@ public class ScanBatch implements CloseableRecordBatch {
public VectorContainer getContainer() {
return container;
}
+
/**
* Verify list of implicit column values is valid input:
* - Either implicit column list is empty;
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/validate/BatchValidator.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/validate/BatchValidator.java
index 2753f55..7b9ff80 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/validate/BatchValidator.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/validate/BatchValidator.java
@@ -20,12 +20,16 @@ package org.apache.drill.exec.physical.impl.validate;
import java.util.IdentityHashMap;
import java.util.Map;
-import org.apache.drill.exec.physical.impl.protocol.OperatorRecordBatch;
+import org.apache.drill.exec.ops.OperatorContext;
+import org.apache.drill.exec.physical.base.PhysicalOperator;
+import org.apache.drill.exec.physical.impl.ScanBatch;
import org.apache.drill.exec.record.RecordBatch;
import org.apache.drill.exec.record.SimpleVectorWrapper;
import org.apache.drill.exec.record.VectorAccessible;
import org.apache.drill.exec.record.VectorContainer;
import org.apache.drill.exec.record.VectorWrapper;
+import org.apache.drill.exec.store.dfs.FormatPlugin;
+import org.apache.drill.exec.store.dfs.easy.EasySubScan;
import org.apache.drill.exec.vector.BitVector;
import org.apache.drill.exec.vector.FixedWidthVector;
import org.apache.drill.exec.vector.NullableVector;
@@ -33,6 +37,7 @@ import org.apache.drill.exec.vector.RepeatedBitVector;
import org.apache.drill.exec.vector.UInt1Vector;
import org.apache.drill.exec.vector.UInt4Vector;
import org.apache.drill.exec.vector.ValueVector;
+import org.apache.drill.exec.vector.VarBinaryVector;
import org.apache.drill.exec.vector.VarCharVector;
import org.apache.drill.exec.vector.VariableWidthVector;
import org.apache.drill.exec.vector.complex.BaseRepeatedValueVector;
@@ -137,9 +142,9 @@ public class BatchValidator {
}
}
- private enum CheckMode { COUNTS, ALL };
+ private enum CheckMode { COUNTS, ALL, NONE, SPECIAL };
- private static final Map<Class<? extends RecordBatch>, CheckMode> checkRules
= buildRules();
+ private static final Map<Class<?>, CheckMode> checkRules = buildRules();
private final ErrorReporter errorReporter;
@@ -153,18 +158,44 @@ public class BatchValidator {
* Over time, this table should include all operators, and thus become
* unnecessary.
*/
- private static Map<Class<? extends RecordBatch>, CheckMode> buildRules() {
- Map<Class<? extends RecordBatch>, CheckMode> rules = new
IdentityHashMap<>();
- rules.put(OperatorRecordBatch.class, CheckMode.ALL);
+ private static Map<Class<?>, CheckMode> buildRules() {
+ final Map<Class<?>, CheckMode> rules = new IdentityHashMap<>();
+ // Operators
+ rules.put(ScanBatch.class, CheckMode.SPECIAL);
+ // Scan types
+ rules.put(EasySubScan.class, CheckMode.ALL);
return rules;
}
+ private static CheckMode lookup(Object subject) {
+ final CheckMode checkMode = checkRules.get(subject.getClass());
+ return checkMode == null ? CheckMode.NONE : checkMode;
+ }
+
+ private static CheckMode checkMode(RecordBatch batch) {
+ final CheckMode checkMode = lookup(batch);
+ if (checkMode != CheckMode.SPECIAL) {
+ return checkMode;
+ }
+ // For Scan, enable readers one-by-one.
+ final ScanBatch scan = (ScanBatch) batch;
+ final OperatorContext opContext = scan.getOperatorContext();
+ final PhysicalOperator opDefn = opContext.getOperatorDefn();
+ final CheckMode opCheckMode = lookup(opDefn);
+ if (opCheckMode != CheckMode.SPECIAL) {
+ return opCheckMode;
+ }
+ final EasySubScan easyScan = (EasySubScan) opDefn;
+ final FormatPlugin plugin = easyScan.getFormatPlugin();
+ return lookup(plugin);
+ }
+
public static boolean validate(RecordBatch batch) {
- CheckMode checkMode = checkRules.get(batch.getClass());
+ final CheckMode checkMode = checkMode(batch);
// If no rule, don't check this batch.
- if (checkMode == null) {
+ if (checkMode == CheckMode.NONE) {
// As work proceeds, might want to log those batches not checked.
// For now, there are too many.
@@ -312,6 +343,8 @@ public class BatchValidator {
if (vector instanceof VarCharVector) {
validateVarCharVector(name, (VarCharVector) vector);
+ } else if (vector instanceof VarBinaryVector) {
+ validateVarBinaryVector(name, (VarBinaryVector) vector);
} else {
logger.debug("Don't know how to validate vector: {} of class {}",
name, vector.getClass().getSimpleName());
@@ -331,6 +364,19 @@ public class BatchValidator {
validateOffsetVector(name + "-offsets", vector.getOffsetVector(), false,
valueCount, dataLength);
}
+ private void validateVarBinaryVector(String name, VarBinaryVector vector) {
+ final int valueCount = vector.getAccessor().getValueCount();
+
+ // Disabled because a large number of operators
+ // set up offset vectors wrongly.
+ if (valueCount == 0) {
+ return;
+ }
+
+ final int dataLength = vector.getBuffer().writerIndex();
+ validateOffsetVector(name + "-offsets", vector.getOffsetVector(), false,
valueCount, dataLength);
+ }
+
private void validateRepeatedVector(String name, BaseRepeatedValueVector
vector) {
ValueVector dataVector = vector.getDataVector();
int dataLength = dataVector.getAccessor().getValueCount();
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/record/VectorContainer.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/record/VectorContainer.java
index c2553c2..4ee50f5 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/record/VectorContainer.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/record/VectorContainer.java
@@ -448,6 +448,11 @@ public class VectorContainer implements VectorAccessible {
return true;
}
+ public void setValueCount(int valueCount) {
+ VectorAccessibleUtilities.setValueCount(this, valueCount);
+ setRecordCount(valueCount);
+ }
+
/**
* Merge two batches to create a single, combined, batch. Vectors
* appear in the order defined by {@link BatchSchema#merge(BatchSchema)}.
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONFormatPlugin.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONFormatPlugin.java
index fea350a..c812999 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONFormatPlugin.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONFormatPlugin.java
@@ -17,15 +17,11 @@
*/
package org.apache.drill.exec.store.easy.json;
-import com.fasterxml.jackson.annotation.JsonInclude;
-import com.fasterxml.jackson.annotation.JsonTypeName;
-import com.fasterxml.jackson.core.JsonFactory;
-import com.fasterxml.jackson.core.JsonGenerator;
-import com.fasterxml.jackson.databind.ObjectMapper;
import java.io.IOException;
import java.io.OutputStream;
import java.util.List;
import java.util.Map;
+
import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.common.logical.FormatPluginConfig;
import org.apache.drill.common.logical.StoragePluginConfig;
@@ -51,9 +47,17 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonTypeName;
+import com.fasterxml.jackson.core.JsonFactory;
+import com.fasterxml.jackson.core.JsonGenerator;
+import com.fasterxml.jackson.databind.ObjectMapper;
public class JSONFormatPlugin extends EasyFormatPlugin<JSONFormatConfig> {
- private static final org.slf4j.Logger logger =
org.slf4j.LoggerFactory.getLogger(JSONFormatPlugin.class);
+ private static final Logger logger =
LoggerFactory.getLogger(JSONFormatPlugin.class);
public static final String DEFAULT_NAME = "json";
private static final boolean IS_COMPRESSIBLE = true;
@@ -77,11 +81,7 @@ public class JSONFormatPlugin extends
EasyFormatPlugin<JSONFormatConfig> {
@Override
public boolean isStatisticsRecordWriter(FragmentContext context, EasyWriter
writer) {
- if (context.getSQLStatementType() == SqlStatementType.ANALYZE) {
- return true;
- } else {
- return false;
- }
+ return context.getSQLStatementType() == SqlStatementType.ANALYZE;
}
@Override
@@ -181,7 +181,7 @@ public class JSONFormatPlugin extends
EasyFormatPlugin<JSONFormatConfig> {
@Override
public int hashCode() {
- final int prime = 31;
+ int prime = 31;
int result = 1;
result = prime * result + ((extensions == null) ? 0 :
extensions.hashCode());
return result;
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/sequencefile/SequenceFileFormatPlugin.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/sequencefile/SequenceFileFormatPlugin.java
index 701c5c5..707bdeb 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/sequencefile/SequenceFileFormatPlugin.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/sequencefile/SequenceFileFormatPlugin.java
@@ -17,13 +17,16 @@
*/
package org.apache.drill.exec.store.easy.sequencefile;
+import java.io.IOException;
+import java.util.List;
+
import org.apache.drill.common.exceptions.ExecutionSetupException;
import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.common.logical.StoragePluginConfig;
import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.physical.base.AbstractGroupScan;
-import org.apache.drill.exec.proto.UserBitShared.CoreOperatorType;
import org.apache.drill.exec.planner.common.DrillStatsTable.TableStatistics;
+import org.apache.drill.exec.proto.UserBitShared.CoreOperatorType;
import org.apache.drill.exec.server.DrillbitContext;
import org.apache.drill.exec.store.RecordReader;
import org.apache.drill.exec.store.RecordWriter;
@@ -38,9 +41,6 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapred.FileSplit;
-import java.io.IOException;
-import java.util.List;
-
public class SequenceFileFormatPlugin extends
EasyFormatPlugin<SequenceFileFormatConfig> {
public SequenceFileFormatPlugin(String name, DrillbitContext context,
Configuration fsConf,
StoragePluginConfig storageConfig) {
@@ -59,7 +59,6 @@ public class SequenceFileFormatPlugin extends
EasyFormatPlugin<SequenceFileForma
return true;
}
-
@Override
public AbstractGroupScan getGroupScan(String userName, FileSelection
selection, List<SchemaPath> columns)
throws IOException {
diff --git
a/exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestDateTypes.java
b/exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestDateTypes.java
index d2d1a12..0a89557 100644
---
a/exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestDateTypes.java
+++
b/exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestDateTypes.java
@@ -23,6 +23,7 @@ import static org.junit.Assert.assertTrue;
import java.util.Iterator;
import java.util.List;
+import org.apache.drill.categories.SlowTest;
import org.apache.drill.categories.VectorTest;
import org.apache.drill.common.util.DrillFileUtils;
import org.apache.drill.exec.client.DrillClient;
@@ -33,251 +34,246 @@ import org.apache.drill.exec.rpc.user.QueryDataBatch;
import org.apache.drill.exec.server.Drillbit;
import org.apache.drill.exec.server.RemoteServiceSet;
import org.apache.drill.exec.vector.ValueVector;
-import org.apache.drill.categories.SlowTest;
-import org.junit.Test;
-
import org.apache.drill.shaded.guava.com.google.common.base.Charsets;
import org.apache.drill.shaded.guava.com.google.common.io.Files;
+import org.junit.Test;
import org.junit.experimental.categories.Category;
-/* This class tests the existing date types. Simply using date types
+/* Tests the existing date types. Simply using date types
* by casting from VarChar, performing basic functions and converting
* back to VarChar.
*/
@Category({SlowTest.class, VectorTest.class})
public class TestDateTypes extends PopUnitTestBase {
-// private static final org.slf4j.Logger logger =
org.slf4j.LoggerFactory.getLogger(TestDateTypes.class);
- @Test
- public void testDate() throws Exception {
- try (RemoteServiceSet serviceSet =
RemoteServiceSet.getLocalServiceSet();
- Drillbit bit = new Drillbit(CONFIG, serviceSet);
- DrillClient client = new DrillClient(CONFIG,
serviceSet.getCoordinator())) {
+ @Test
+ public void testDate() throws Exception {
+ try (RemoteServiceSet serviceSet = RemoteServiceSet.getLocalServiceSet();
+ Drillbit bit = new Drillbit(CONFIG, serviceSet);
+ DrillClient client = new DrillClient(CONFIG,
serviceSet.getCoordinator())) {
- // run query.
- bit.run();
- client.connect();
- List<QueryDataBatch> results =
client.runQuery(org.apache.drill.exec.proto.UserBitShared.QueryType.PHYSICAL,
-
Files.asCharSource(DrillFileUtils.getResourceAsFile("/record/vector/test_date.json"),
Charsets.UTF_8).read()
- .replace("#{TEST_FILE}",
"/test_simple_date.json"));
+ // run query.
+ bit.run();
+ client.connect();
+ List<QueryDataBatch> results =
client.runQuery(org.apache.drill.exec.proto.UserBitShared.QueryType.PHYSICAL,
+
Files.asCharSource(DrillFileUtils.getResourceAsFile("/record/vector/test_date.json"),
Charsets.UTF_8).read()
+ .replace("#{TEST_FILE}", "/test_simple_date.json"));
- RecordBatchLoader batchLoader = new
RecordBatchLoader(bit.getContext().getAllocator());
+ RecordBatchLoader batchLoader = new
RecordBatchLoader(bit.getContext().getAllocator());
- QueryDataBatch batch = results.get(0);
- assertTrue(batchLoader.load(batch.getHeader().getDef(),
batch.getData()));
+ QueryDataBatch batch = results.get(0);
+ assertTrue(batchLoader.load(batch.getHeader().getDef(),
batch.getData()));
- for (VectorWrapper<?> v : batchLoader) {
+ for (VectorWrapper<?> v : batchLoader) {
- ValueVector.Accessor accessor =
v.getValueVector().getAccessor();
+ ValueVector.Accessor accessor = v.getValueVector().getAccessor();
- assertEquals((accessor.getObject(0).toString()),
("1970-01-02"));
- assertEquals((accessor.getObject(1).toString()),
("2008-12-28"));
- assertEquals((accessor.getObject(2).toString()),
("2000-02-27"));
- }
+ assertEquals((accessor.getObject(0).toString()), ("1970-01-02"));
+ assertEquals((accessor.getObject(1).toString()), ("2008-12-28"));
+ assertEquals((accessor.getObject(2).toString()), ("2000-02-27"));
+ }
- batchLoader.clear();
- for(QueryDataBatch b : results){
- b.release();
- }
- }
+ batchLoader.clear();
+ for(QueryDataBatch b : results){
+ b.release();
+ }
}
+ }
- @Test
- public void testSortDate() throws Exception {
- try (RemoteServiceSet serviceSet =
RemoteServiceSet.getLocalServiceSet();
- Drillbit bit = new Drillbit(CONFIG, serviceSet);
- DrillClient client = new DrillClient(CONFIG,
serviceSet.getCoordinator())) {
+ @Test
+ public void testSortDate() throws Exception {
+ try (RemoteServiceSet serviceSet = RemoteServiceSet.getLocalServiceSet();
+ Drillbit bit = new Drillbit(CONFIG, serviceSet);
+ DrillClient client = new DrillClient(CONFIG,
serviceSet.getCoordinator())) {
- // run query.
- bit.run();
- client.connect();
- List<QueryDataBatch> results =
client.runQuery(org.apache.drill.exec.proto.UserBitShared.QueryType.PHYSICAL,
-
Files.asCharSource(DrillFileUtils.getResourceAsFile("/record/vector/test_sort_date.json"),
Charsets.UTF_8).read()
- .replace("#{TEST_FILE}",
"/test_simple_date.json"));
+ // run query.
+ bit.run();
+ client.connect();
+ List<QueryDataBatch> results =
client.runQuery(org.apache.drill.exec.proto.UserBitShared.QueryType.PHYSICAL,
+
Files.asCharSource(DrillFileUtils.getResourceAsFile("/record/vector/test_sort_date.json"),
Charsets.UTF_8).read()
+ .replace("#{TEST_FILE}", "/test_simple_date.json"));
- RecordBatchLoader batchLoader = new
RecordBatchLoader(bit.getContext().getAllocator());
+ RecordBatchLoader batchLoader = new
RecordBatchLoader(bit.getContext().getAllocator());
- QueryDataBatch batch = results.get(1);
- assertTrue(batchLoader.load(batch.getHeader().getDef(),
batch.getData()));
+ QueryDataBatch batch = results.get(1);
+ assertTrue(batchLoader.load(batch.getHeader().getDef(),
batch.getData()));
- for (VectorWrapper<?> v : batchLoader) {
+ for (VectorWrapper<?> v : batchLoader) {
- ValueVector.Accessor accessor =
v.getValueVector().getAccessor();
+ ValueVector.Accessor accessor = v.getValueVector().getAccessor();
- assertEquals((accessor.getObject(0).toString()), "1970-01-02");
- assertEquals((accessor.getObject(1).toString()), "2000-02-27");
- assertEquals((accessor.getObject(2).toString()), "2008-12-28");
- }
+ assertEquals((accessor.getObject(0).toString()), "1970-01-02");
+ assertEquals((accessor.getObject(1).toString()), "2000-02-27");
+ assertEquals((accessor.getObject(2).toString()), "2008-12-28");
+ }
- batchLoader.clear();
- for(QueryDataBatch b : results){
- b.release();
- }
- }
+ batchLoader.clear();
+ for(QueryDataBatch b : results){
+ b.release();
+ }
}
+ }
- @Test
- public void testTimeStamp() throws Exception {
- try (RemoteServiceSet serviceSet =
RemoteServiceSet.getLocalServiceSet();
- Drillbit bit = new Drillbit(CONFIG, serviceSet);
- DrillClient client = new DrillClient(CONFIG,
serviceSet.getCoordinator())) {
+ @Test
+ public void testTimeStamp() throws Exception {
+ try (RemoteServiceSet serviceSet = RemoteServiceSet.getLocalServiceSet();
+ Drillbit bit = new Drillbit(CONFIG, serviceSet);
+ DrillClient client = new DrillClient(CONFIG,
serviceSet.getCoordinator())) {
- // run query.
- bit.run();
- client.connect();
- List<QueryDataBatch> results =
client.runQuery(org.apache.drill.exec.proto.UserBitShared.QueryType.PHYSICAL,
-
Files.asCharSource(DrillFileUtils.getResourceAsFile("/record/vector/test_timestamp.json"),
Charsets.UTF_8).read()
- .replace("#{TEST_FILE}",
"/test_simple_date.json"));
+ // run query.
+ bit.run();
+ client.connect();
+ List<QueryDataBatch> results =
client.runQuery(org.apache.drill.exec.proto.UserBitShared.QueryType.PHYSICAL,
+
Files.asCharSource(DrillFileUtils.getResourceAsFile("/record/vector/test_timestamp.json"),
Charsets.UTF_8).read()
+ .replace("#{TEST_FILE}", "/test_simple_date.json"));
- RecordBatchLoader batchLoader = new
RecordBatchLoader(bit.getContext().getAllocator());
+ RecordBatchLoader batchLoader = new
RecordBatchLoader(bit.getContext().getAllocator());
- QueryDataBatch batch = results.get(0);
- assertTrue(batchLoader.load(batch.getHeader().getDef(),
batch.getData()));
+ QueryDataBatch batch = results.get(0);
+ assertTrue(batchLoader.load(batch.getHeader().getDef(),
batch.getData()));
- for (VectorWrapper<?> v : batchLoader) {
+ for (VectorWrapper<?> v : batchLoader) {
- ValueVector.Accessor accessor =
v.getValueVector().getAccessor();
+ ValueVector.Accessor accessor = v.getValueVector().getAccessor();
- assertEquals(accessor.getObject(0).toString(),"1970-01-02
10:20:33.000");
- assertEquals(accessor.getObject(1).toString(),"2008-12-28
11:34:00.129");
- assertEquals(accessor.getObject(2).toString(), "2000-02-27
14:24:00.000");
- }
+ assertEquals(accessor.getObject(0).toString(),"1970-01-02
10:20:33.000");
+ assertEquals(accessor.getObject(1).toString(),"2008-12-28
11:34:00.129");
+ assertEquals(accessor.getObject(2).toString(), "2000-02-27
14:24:00.000");
+ }
- batchLoader.clear();
- for(QueryDataBatch b : results){
- b.release();
- }
- }
+ batchLoader.clear();
+ for(QueryDataBatch b : results){
+ b.release();
+ }
}
-
- @Test
- public void testInterval() throws Exception {
- try (RemoteServiceSet serviceSet =
RemoteServiceSet.getLocalServiceSet();
- Drillbit bit = new Drillbit(CONFIG, serviceSet);
- DrillClient client = new DrillClient(CONFIG,
serviceSet.getCoordinator())) {
-
- // run query.
- bit.run();
- client.connect();
- List<QueryDataBatch> results =
client.runQuery(org.apache.drill.exec.proto.UserBitShared.QueryType.PHYSICAL,
-
Files.asCharSource(DrillFileUtils.getResourceAsFile("/record/vector/test_interval.json"),
Charsets.UTF_8).read()
- .replace("#{TEST_FILE}",
"/test_simple_interval.json"));
-
- RecordBatchLoader batchLoader = new
RecordBatchLoader(bit.getContext().getAllocator());
-
- QueryDataBatch batch = results.get(0);
- assertTrue(batchLoader.load(batch.getHeader().getDef(),
batch.getData()));
-
- Iterator<VectorWrapper<?>> itr = batchLoader.iterator();
-
- ValueVector.Accessor accessor =
itr.next().getValueVector().getAccessor();
-
- // Check the interval type
- assertEquals((accessor.getObject(0).toString()), ("2 years 2
months 1 day 1:20:35.0"));
- assertEquals((accessor.getObject(1).toString()), ("2 years 2
months 0 days 0:0:0.0"));
- assertEquals((accessor.getObject(2).toString()), ("0 years 0
months 0 days 1:20:35.0"));
- assertEquals((accessor.getObject(3).toString()),("2 years 2 months
1 day 1:20:35.897"));
- assertEquals((accessor.getObject(4).toString()), ("0 years 0
months 0 days 0:0:35.4"));
- assertEquals((accessor.getObject(5).toString()), ("1 year 10
months 1 day 0:-39:-25.0"));
-
- accessor = itr.next().getValueVector().getAccessor();
-
- // Check the interval year type
- assertEquals((accessor.getObject(0).toString()), ("2 years 2
months "));
- assertEquals((accessor.getObject(1).toString()), ("2 years 2
months "));
- assertEquals((accessor.getObject(2).toString()), ("0 years 0
months "));
- assertEquals((accessor.getObject(3).toString()), ("2 years 2
months "));
- assertEquals((accessor.getObject(4).toString()), ("0 years 0
months "));
- assertEquals((accessor.getObject(5).toString()), ("1 year 10
months "));
-
-
- accessor = itr.next().getValueVector().getAccessor();
-
- // Check the interval day type
- assertEquals((accessor.getObject(0).toString()), ("1 day
1:20:35.0"));
- assertEquals((accessor.getObject(1).toString()), ("0 days
0:0:0.0"));
- assertEquals((accessor.getObject(2).toString()), ("0 days
1:20:35.0"));
- assertEquals((accessor.getObject(3).toString()), ("1 day
1:20:35.897"));
- assertEquals((accessor.getObject(4).toString()), ("0 days
0:0:35.4"));
- assertEquals((accessor.getObject(5).toString()), ("1 day
0:-39:-25.0"));
-
- batchLoader.clear();
- for(QueryDataBatch b : results){
- b.release();
- }
- }
+ }
+
+ @Test
+ public void testInterval() throws Exception {
+ try (RemoteServiceSet serviceSet = RemoteServiceSet.getLocalServiceSet();
+ Drillbit bit = new Drillbit(CONFIG, serviceSet);
+ DrillClient client = new DrillClient(CONFIG,
serviceSet.getCoordinator())) {
+
+ // run query.
+ bit.run();
+ client.connect();
+ List<QueryDataBatch> results =
client.runQuery(org.apache.drill.exec.proto.UserBitShared.QueryType.PHYSICAL,
+
Files.asCharSource(DrillFileUtils.getResourceAsFile("/record/vector/test_interval.json"),
Charsets.UTF_8).read()
+ .replace("#{TEST_FILE}", "/test_simple_interval.json"));
+
+ RecordBatchLoader batchLoader = new
RecordBatchLoader(bit.getContext().getAllocator());
+
+ QueryDataBatch batch = results.get(0);
+ assertTrue(batchLoader.load(batch.getHeader().getDef(),
batch.getData()));
+
+ Iterator<VectorWrapper<?>> itr = batchLoader.iterator();
+
+ ValueVector.Accessor accessor =
itr.next().getValueVector().getAccessor();
+
+ // Check the interval type
+ assertEquals((accessor.getObject(0).toString()), ("2 years 2 months 1
day 1:20:35.0"));
+ assertEquals((accessor.getObject(1).toString()), ("2 years 2 months 0
days 0:0:0.0"));
+ assertEquals((accessor.getObject(2).toString()), ("0 years 0 months 0
days 1:20:35.0"));
+ assertEquals((accessor.getObject(3).toString()),("2 years 2 months 1 day
1:20:35.897"));
+ assertEquals((accessor.getObject(4).toString()), ("0 years 0 months 0
days 0:0:35.4"));
+ assertEquals((accessor.getObject(5).toString()), ("1 year 10 months 1
day 0:-39:-25.0"));
+
+ accessor = itr.next().getValueVector().getAccessor();
+
+ // Check the interval year type
+ assertEquals((accessor.getObject(0).toString()), ("2 years 2 months "));
+ assertEquals((accessor.getObject(1).toString()), ("2 years 2 months "));
+ assertEquals((accessor.getObject(2).toString()), ("0 years 0 months "));
+ assertEquals((accessor.getObject(3).toString()), ("2 years 2 months "));
+ assertEquals((accessor.getObject(4).toString()), ("0 years 0 months "));
+ assertEquals((accessor.getObject(5).toString()), ("1 year 10 months "));
+
+
+ accessor = itr.next().getValueVector().getAccessor();
+
+ // Check the interval day type
+ assertEquals((accessor.getObject(0).toString()), ("1 day 1:20:35.0"));
+ assertEquals((accessor.getObject(1).toString()), ("0 days 0:0:0.0"));
+ assertEquals((accessor.getObject(2).toString()), ("0 days 1:20:35.0"));
+ assertEquals((accessor.getObject(3).toString()), ("1 day 1:20:35.897"));
+ assertEquals((accessor.getObject(4).toString()), ("0 days 0:0:35.4"));
+ assertEquals((accessor.getObject(5).toString()), ("1 day 0:-39:-25.0"));
+
+ batchLoader.clear();
+ for(QueryDataBatch b : results){
+ b.release();
+ }
}
+ }
- @Test
- public void testLiterals() throws Exception {
- try (RemoteServiceSet serviceSet =
RemoteServiceSet.getLocalServiceSet();
- Drillbit bit = new Drillbit(CONFIG, serviceSet);
- DrillClient client = new DrillClient(CONFIG,
serviceSet.getCoordinator())) {
+ @Test
+ public void testLiterals() throws Exception {
+ try (RemoteServiceSet serviceSet = RemoteServiceSet.getLocalServiceSet();
+ Drillbit bit = new Drillbit(CONFIG, serviceSet);
+ DrillClient client = new DrillClient(CONFIG,
serviceSet.getCoordinator())) {
- // run query.
- bit.run();
- client.connect();
- List<QueryDataBatch> results =
client.runQuery(org.apache.drill.exec.proto.UserBitShared.QueryType.PHYSICAL,
-
Files.asCharSource(DrillFileUtils.getResourceAsFile("/record/vector/test_all_date_literals.json"),
Charsets.UTF_8).read()
- .replace("#{TEST_FILE}",
"/test_simple_date.json"));
+ // run query.
+ bit.run();
+ client.connect();
+ List<QueryDataBatch> results =
client.runQuery(org.apache.drill.exec.proto.UserBitShared.QueryType.PHYSICAL,
+
Files.asCharSource(DrillFileUtils.getResourceAsFile("/record/vector/test_all_date_literals.json"),
Charsets.UTF_8).read()
+ .replace("#{TEST_FILE}", "/test_simple_date.json"));
- RecordBatchLoader batchLoader = new
RecordBatchLoader(bit.getContext().getAllocator());
+ RecordBatchLoader batchLoader = new
RecordBatchLoader(bit.getContext().getAllocator());
- QueryDataBatch batch = results.get(0);
- assertTrue(batchLoader.load(batch.getHeader().getDef(),
batch.getData()));
+ QueryDataBatch batch = results.get(0);
+ assertTrue(batchLoader.load(batch.getHeader().getDef(),
batch.getData()));
- String result[] = {"2008-02-27",
- "2008-02-27 01:02:03.000",
- "10:11:13.999",
- "2 years 2 months 3 days 0:1:3.89"};
+ String result[] = {"2008-02-27",
+ "2008-02-27 01:02:03.000",
+ "10:11:13.999",
+ "2 years 2 months 3 days 0:1:3.89"};
- int idx = 0;
+ int idx = 0;
- for (VectorWrapper<?> v : batchLoader) {
+ for (VectorWrapper<?> v : batchLoader) {
- ValueVector.Accessor accessor =
v.getValueVector().getAccessor();
+ ValueVector.Accessor accessor = v.getValueVector().getAccessor();
- assertEquals((accessor.getObject(0).toString()),
(result[idx++]));
- }
+ assertEquals((accessor.getObject(0).toString()), (result[idx++]));
+ }
- batchLoader.clear();
- for(QueryDataBatch b : results){
- b.release();
- }
- }
+ batchLoader.clear();
+ for(QueryDataBatch b : results){
+ b.release();
+ }
}
+ }
- @Test
- public void testDateAdd() throws Exception {
- try (RemoteServiceSet serviceSet =
RemoteServiceSet.getLocalServiceSet();
- Drillbit bit = new Drillbit(CONFIG, serviceSet);
- DrillClient client = new DrillClient(CONFIG,
serviceSet.getCoordinator())) {
-
- // run query.
- bit.run();
- client.connect();
- List<QueryDataBatch> results =
client.runQuery(org.apache.drill.exec.proto.UserBitShared.QueryType.PHYSICAL,
-
Files.asCharSource(DrillFileUtils.getResourceAsFile("/record/vector/test_date_add.json"),
Charsets.UTF_8).read()
- .replace("#{TEST_FILE}",
"/test_simple_date.json"));
-
- RecordBatchLoader batchLoader = new
RecordBatchLoader(bit.getContext().getAllocator());
+ @Test
+ public void testDateAdd() throws Exception {
+ try (RemoteServiceSet serviceSet = RemoteServiceSet.getLocalServiceSet();
+ Drillbit bit = new Drillbit(CONFIG, serviceSet);
+ DrillClient client = new DrillClient(CONFIG,
serviceSet.getCoordinator())) {
- QueryDataBatch batch = results.get(0);
- assertTrue(batchLoader.load(batch.getHeader().getDef(),
batch.getData()));
+ // run query.
+ bit.run();
+ client.connect();
+ List<QueryDataBatch> results =
client.runQuery(org.apache.drill.exec.proto.UserBitShared.QueryType.PHYSICAL,
+
Files.asCharSource(DrillFileUtils.getResourceAsFile("/record/vector/test_date_add.json"),
Charsets.UTF_8).read()
+ .replace("#{TEST_FILE}", "/test_simple_date.json"));
- for (VectorWrapper<?> v : batchLoader) {
+ RecordBatchLoader batchLoader = new
RecordBatchLoader(bit.getContext().getAllocator());
- ValueVector.Accessor accessor =
v.getValueVector().getAccessor();
+ QueryDataBatch batch = results.get(0);
+ assertTrue(batchLoader.load(batch.getHeader().getDef(),
batch.getData()));
- assertEquals((accessor.getObject(0).toString()), ("2008-03-27
00:00:00.000"));
+ for (VectorWrapper<?> v : batchLoader) {
+ ValueVector.Accessor accessor = v.getValueVector().getAccessor();
- }
+ assertEquals((accessor.getObject(0).toString()), ("2008-03-27
00:00:00.000"));
+ }
- batchLoader.clear();
- for(QueryDataBatch b : results){
- b.release();
- }
- }
+ batchLoader.clear();
+ for(QueryDataBatch b : results){
+ b.release();
+ }
}
+ }
}
diff --git a/exec/java-exec/src/test/resources/parquet/alltypes_repeated.json
b/exec/java-exec/src/test/resources/parquet/alltypes_repeated.json
index 141c185..7ebf037 100644
--- a/exec/java-exec/src/test/resources/parquet/alltypes_repeated.json
+++ b/exec/java-exec/src/test/resources/parquet/alltypes_repeated.json
@@ -108,5 +108,5 @@
"VARBINARY_col" : false,
"VARCHAR_col" : [ "a string", "asdf", ""],
"VAR16CHAR_col" : false,
- "BIT_col" : [ true, true, true]
-}
\ No newline at end of file
+ "BIT_col" : [ true, true, true]
+}
