[
https://issues.apache.org/jira/browse/CARBONDATA-2591?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
xubo245 updated CARBONDATA-2591:
--------------------------------
Description:
SDK CarbonReader support filter
Code:
{code:java}
@Test
public void testReadWithFilterNonTransactional() throws IOException,
InterruptedException {
String path = "./testWriteFiles";
FileUtils.deleteDirectory(new File(path));
Field[] fields = new Field[2];
fields[0] = new Field("name", DataTypes.STRING);
fields[1] = new Field("age", DataTypes.INT);
TestUtil.writeFilesAndVerify(new Schema(fields), path, false, false);
ColumnSchema columnSchema = new ColumnSchema();
columnSchema.setColumnar(true);
columnSchema.setColumnName("name");
columnSchema.setColumnUniqueId(UUID.randomUUID().toString());
columnSchema.setDataType(DataTypes.STRING);
columnSchema.setDimensionColumn(true);
List<Encoding> encodingList = new ArrayList<>();
encodingList.add(Encoding.IMPLICIT);
columnSchema.setEncodingList(encodingList);
CarbonColumn carbonColumn = new CarbonColumn(columnSchema, 0, 0);
ColumnExpression columnExpression = new ColumnExpression("name",
DataTypes.STRING);
columnExpression.setCarbonColumn(carbonColumn);
EqualToExpression equalToExpression = new
EqualToExpression(columnExpression,
new LiteralExpression("robot1", DataTypes.STRING));
CarbonReader reader = CarbonReader
.builder(path, "_temp")
.isTransactionalTable(false)
.projection(new String[]{"name", "age"})
.filter(equalToExpression)
.build();
// expected output after sorting
String[] name = new String[200];
Integer[] age = new Integer[200];
for (int i = 0; i < 200; i++) {
name[i] = "robot" + (i / 10);
age[i] = i;
}
int i = 0;
while (reader.hasNext()) {
Object[] row = (Object[]) reader.readNextRow();
// Default sort column is applied for dimensions. So, need to validate
accordingly
assert ("robot1".equals(row[0]));
System.out.println(row[0] + "\t" + row[1]);
i++;
}
Assert.assertEquals(i, 20);
reader.close();
FileUtils.deleteDirectory(new File(path));
}
{code}
Error:
{code:java}
log4j:WARN No appenders could be found for logger
(org.apache.carbondata.core.util.CarbonProperties).
log4j:WARN Please initialize the log4j system properly.
log4j:WARN See http://logging.apache.org/log4j/1.2/faq.html#noconfig for more
info.
java.lang.RuntimeException: Error while resolving filter expression
at
org.apache.carbondata.core.metadata.schema.table.CarbonTable.resolveFilter(CarbonTable.java:909)
at
org.apache.carbondata.hadoop.api.CarbonFileInputFormat.getSplits(CarbonFileInputFormat.java:128)
at
org.apache.carbondata.sdk.file.CarbonReaderBuilder.build(CarbonReaderBuilder.java:200)
at
org.apache.carbondata.sdk.file.CarbonReaderTest.testReadWithFilterNonTransactional(CarbonReaderTest.java:189)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at junit.framework.TestCase.runTest(TestCase.java:176)
at junit.framework.TestCase.runBare(TestCase.java:141)
at junit.framework.TestResult$1.protect(TestResult.java:122)
at junit.framework.TestResult.runProtected(TestResult.java:142)
at junit.framework.TestResult.run(TestResult.java:125)
at junit.framework.TestCase.run(TestCase.java:129)
at junit.framework.TestSuite.runTest(TestSuite.java:255)
at junit.framework.TestSuite.run(TestSuite.java:250)
at
org.junit.internal.runners.JUnit38ClassRunner.run(JUnit38ClassRunner.java:84)
at org.junit.runner.JUnitCore.run(JUnitCore.java:160)
at
com.intellij.junit4.JUnit4IdeaTestRunner.startRunnerWithArgs(JUnit4IdeaTestRunner.java:68)
at
com.intellij.rt.execution.junit.IdeaTestRunner$Repeater.startRunnerWithArgs(IdeaTestRunner.java:47)
at
com.intellij.rt.execution.junit.JUnitStarter.prepareStreamsAndStart(JUnitStarter.java:242)
at
com.intellij.rt.execution.junit.JUnitStarter.main(JUnitStarter.java:70)
Caused by: java.lang.NullPointerException
at
org.apache.carbondata.core.scan.filter.FilterExpressionProcessor.getFilterResolverBasedOnExpressionType(FilterExpressionProcessor.java:353)
at
org.apache.carbondata.core.scan.filter.FilterExpressionProcessor.createFilterResolverTree(FilterExpressionProcessor.java:284)
at
org.apache.carbondata.core.scan.filter.FilterExpressionProcessor.getFilterResolvertree(FilterExpressionProcessor.java:233)
at
org.apache.carbondata.core.scan.filter.FilterExpressionProcessor.getFilterResolver(FilterExpressionProcessor.java:84)
at
org.apache.carbondata.core.metadata.schema.table.CarbonTable.resolveFilter(CarbonTable.java:906)
... 21 more
{code}
was:SDK CarbonReader support filter
> SDK CarbonReader support filter
> -------------------------------
>
> Key: CARBONDATA-2591
> URL: https://issues.apache.org/jira/browse/CARBONDATA-2591
> Project: CarbonData
> Issue Type: Improvement
> Reporter: xubo245
> Assignee: xubo245
> Priority: Major
> Time Spent: 40m
> Remaining Estimate: 0h
>
> SDK CarbonReader support filter
> Code:
> {code:java}
> @Test
> public void testReadWithFilterNonTransactional() throws IOException,
> InterruptedException {
> String path = "./testWriteFiles";
> FileUtils.deleteDirectory(new File(path));
> Field[] fields = new Field[2];
> fields[0] = new Field("name", DataTypes.STRING);
> fields[1] = new Field("age", DataTypes.INT);
> TestUtil.writeFilesAndVerify(new Schema(fields), path, false, false);
> ColumnSchema columnSchema = new ColumnSchema();
> columnSchema.setColumnar(true);
> columnSchema.setColumnName("name");
> columnSchema.setColumnUniqueId(UUID.randomUUID().toString());
> columnSchema.setDataType(DataTypes.STRING);
> columnSchema.setDimensionColumn(true);
> List<Encoding> encodingList = new ArrayList<>();
> encodingList.add(Encoding.IMPLICIT);
> columnSchema.setEncodingList(encodingList);
> CarbonColumn carbonColumn = new CarbonColumn(columnSchema, 0, 0);
> ColumnExpression columnExpression = new ColumnExpression("name",
> DataTypes.STRING);
> columnExpression.setCarbonColumn(carbonColumn);
> EqualToExpression equalToExpression = new
> EqualToExpression(columnExpression,
> new LiteralExpression("robot1", DataTypes.STRING));
> CarbonReader reader = CarbonReader
> .builder(path, "_temp")
> .isTransactionalTable(false)
> .projection(new String[]{"name", "age"})
> .filter(equalToExpression)
> .build();
> // expected output after sorting
> String[] name = new String[200];
> Integer[] age = new Integer[200];
> for (int i = 0; i < 200; i++) {
> name[i] = "robot" + (i / 10);
> age[i] = i;
> }
> int i = 0;
> while (reader.hasNext()) {
> Object[] row = (Object[]) reader.readNextRow();
> // Default sort column is applied for dimensions. So, need to validate
> accordingly
> assert ("robot1".equals(row[0]));
> System.out.println(row[0] + "\t" + row[1]);
> i++;
> }
> Assert.assertEquals(i, 20);
> reader.close();
> FileUtils.deleteDirectory(new File(path));
> }
> {code}
> Error:
> {code:java}
> log4j:WARN No appenders could be found for logger
> (org.apache.carbondata.core.util.CarbonProperties).
> log4j:WARN Please initialize the log4j system properly.
> log4j:WARN See http://logging.apache.org/log4j/1.2/faq.html#noconfig for more
> info.
> java.lang.RuntimeException: Error while resolving filter expression
> at
> org.apache.carbondata.core.metadata.schema.table.CarbonTable.resolveFilter(CarbonTable.java:909)
> at
> org.apache.carbondata.hadoop.api.CarbonFileInputFormat.getSplits(CarbonFileInputFormat.java:128)
> at
> org.apache.carbondata.sdk.file.CarbonReaderBuilder.build(CarbonReaderBuilder.java:200)
> at
> org.apache.carbondata.sdk.file.CarbonReaderTest.testReadWithFilterNonTransactional(CarbonReaderTest.java:189)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:498)
> at junit.framework.TestCase.runTest(TestCase.java:176)
> at junit.framework.TestCase.runBare(TestCase.java:141)
> at junit.framework.TestResult$1.protect(TestResult.java:122)
> at junit.framework.TestResult.runProtected(TestResult.java:142)
> at junit.framework.TestResult.run(TestResult.java:125)
> at junit.framework.TestCase.run(TestCase.java:129)
> at junit.framework.TestSuite.runTest(TestSuite.java:255)
> at junit.framework.TestSuite.run(TestSuite.java:250)
> at
> org.junit.internal.runners.JUnit38ClassRunner.run(JUnit38ClassRunner.java:84)
> at org.junit.runner.JUnitCore.run(JUnitCore.java:160)
> at
> com.intellij.junit4.JUnit4IdeaTestRunner.startRunnerWithArgs(JUnit4IdeaTestRunner.java:68)
> at
> com.intellij.rt.execution.junit.IdeaTestRunner$Repeater.startRunnerWithArgs(IdeaTestRunner.java:47)
> at
> com.intellij.rt.execution.junit.JUnitStarter.prepareStreamsAndStart(JUnitStarter.java:242)
> at
> com.intellij.rt.execution.junit.JUnitStarter.main(JUnitStarter.java:70)
> Caused by: java.lang.NullPointerException
> at
> org.apache.carbondata.core.scan.filter.FilterExpressionProcessor.getFilterResolverBasedOnExpressionType(FilterExpressionProcessor.java:353)
> at
> org.apache.carbondata.core.scan.filter.FilterExpressionProcessor.createFilterResolverTree(FilterExpressionProcessor.java:284)
> at
> org.apache.carbondata.core.scan.filter.FilterExpressionProcessor.getFilterResolvertree(FilterExpressionProcessor.java:233)
> at
> org.apache.carbondata.core.scan.filter.FilterExpressionProcessor.getFilterResolver(FilterExpressionProcessor.java:84)
> at
> org.apache.carbondata.core.metadata.schema.table.CarbonTable.resolveFilter(CarbonTable.java:906)
> ... 21 more
> {code}
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
