This is an automated email from the ASF dual-hosted git repository.
jackylk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/carbondata.git
The following commit(s) were added to refs/heads/master by this push:
new 05f02bd [CARBONDATA-3342] Fix the IllegalArgumentException when using
filter and result is null.
05f02bd is described below
commit 05f02bde570ec94ee977a6732327773530cc5253
Author: xubo245 <xub...@huawei.com>
AuthorDate: Wed Mar 20 22:54:20 2019 +0800
[CARBONDATA-3342] Fix the IllegalArgumentException when using filter and
result is null.
This PR fixs the IllegalArgumentException when using filter and result is
null.
This closes #3273
---
.../hadoop/util/CarbonVectorizedRecordReader.java | 3 ++
.../apache/carbondata/sdk/file/CarbonReader.java | 22 +++++++++++----
.../apache/carbondata/sdk/file/utils/SDKUtil.java | 4 +--
.../carbondata/sdk/file/CarbonReaderTest.java | 32 ++++++++++++++++++++++
4 files changed, 53 insertions(+), 8 deletions(-)
diff --git
a/hadoop/src/main/java/org/apache/carbondata/hadoop/util/CarbonVectorizedRecordReader.java
b/hadoop/src/main/java/org/apache/carbondata/hadoop/util/CarbonVectorizedRecordReader.java
index 132ea6a..54f4bf7 100644
---
a/hadoop/src/main/java/org/apache/carbondata/hadoop/util/CarbonVectorizedRecordReader.java
+++
b/hadoop/src/main/java/org/apache/carbondata/hadoop/util/CarbonVectorizedRecordReader.java
@@ -135,6 +135,9 @@ public class CarbonVectorizedRecordReader extends
AbstractRecordReader<Object> {
if (iterator.hasNext()) {
iterator.processNextBatch(carbonColumnarBatch);
numBatched = carbonColumnarBatch.getActualSize();
+ if (numBatched == 0) {
+ return nextBatch();
+ }
batchIdx = 0;
return true;
}
diff --git
a/sdk/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReader.java
b/sdk/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReader.java
index dcbbc9b..204c36d 100644
--- a/sdk/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReader.java
+++ b/sdk/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReader.java
@@ -57,19 +57,23 @@ public class CarbonReader<T> {
* Call {@link #builder(String)} to construct an instance
*/
CarbonReader(List<RecordReader<Void, T>> readers) {
- if (readers.size() == 0) {
- throw new IllegalArgumentException("no reader");
- }
this.initialise = true;
this.readers = readers;
this.index = 0;
- this.currentReader = readers.get(0);
+ if (0 == readers.size()) {
+ this.currentReader = null;
+ } else {
+ this.currentReader = readers.get(0);
+ }
}
/**
* Return true if has next row
*/
public boolean hasNext() throws IOException, InterruptedException {
+ if (0 == readers.size() || currentReader == null) {
+ return false;
+ }
validateReader();
if (currentReader.nextKeyValue()) {
return true;
@@ -87,9 +91,13 @@ public class CarbonReader<T> {
readers.set(index, null);
index++;
currentReader = readers.get(index);
- return currentReader.nextKeyValue();
+ boolean hasNext = currentReader.nextKeyValue();
+ if (hasNext) {
+ return true;
+ }
}
}
+ return false;
}
/**
@@ -242,7 +250,9 @@ public class CarbonReader<T> {
CarbonProperties.getInstance()
.addProperty(CarbonCommonConstants.DETAIL_QUERY_BATCH_SIZE,
String.valueOf(CarbonCommonConstants.DETAIL_QUERY_BATCH_SIZE_DEFAULT));
- this.currentReader.close();
+ if (null != this.currentReader) {
+ this.currentReader.close();
+ }
this.initialise = false;
}
diff --git
a/sdk/sdk/src/main/java/org/apache/carbondata/sdk/file/utils/SDKUtil.java
b/sdk/sdk/src/main/java/org/apache/carbondata/sdk/file/utils/SDKUtil.java
index 7f9fe88..0362374 100644
--- a/sdk/sdk/src/main/java/org/apache/carbondata/sdk/file/utils/SDKUtil.java
+++ b/sdk/sdk/src/main/java/org/apache/carbondata/sdk/file/utils/SDKUtil.java
@@ -26,8 +26,8 @@ import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.hadoop.conf.Configuration;
public class SDKUtil {
- public static ArrayList listFiles(String sourceImageFolder, final String
suf) {
- return listFiles(sourceImageFolder, suf, new Configuration(true));
+ public static ArrayList listFiles(String sourceFolder, final String suf) {
+ return listFiles(sourceFolder, suf, new Configuration(true));
}
public static ArrayList listFiles(String sourceImageFolder,
diff --git
a/sdk/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonReaderTest.java
b/sdk/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonReaderTest.java
index 15032c2..b108b47 100644
--- a/sdk/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonReaderTest.java
+++ b/sdk/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonReaderTest.java
@@ -2528,4 +2528,36 @@ public class CarbonReaderTest extends TestCase {
FileUtils.deleteDirectory(new File(path));
}
+ @Test
+ public void testReadWithFilterNonResult() throws IOException,
InterruptedException {
+ String path = "./testWriteFiles";
+ FileUtils.deleteDirectory(new File(path));
+ Field[] fields = new Field[2];
+ fields[0] = new Field("name", DataTypes.STRING);
+ fields[1] = new Field("age", DataTypes.INT);
+
+ TestUtil.writeFilesAndVerify(200, new Schema(fields), path);
+
+ ColumnExpression columnExpression = new ColumnExpression("age",
DataTypes.INT);
+
+ EqualToExpression equalToExpression = new
EqualToExpression(columnExpression,
+ new LiteralExpression("-11", DataTypes.INT));
+ CarbonReader reader = CarbonReader
+ .builder(path, "_temp")
+ .projection(new String[]{"name", "age"})
+ .filter(equalToExpression)
+ .build();
+
+ int i = 0;
+ while (reader.hasNext()) {
+ Assert.assertTrue(false);
+ i++;
+ }
+ Assert.assertEquals(i, 0);
+
+ reader.close();
+
+ FileUtils.deleteDirectory(new File(path));
+ }
+
}
