[
https://issues.apache.org/jira/browse/CARBONDATA-2491?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
xubo245 updated CARBONDATA-2491:
--------------------------------
Description:
Test code in org.apache.carbondata.sdk.file.CarbonReaderTest:
{code:java}
@Test
public void testWriteAndReadFiles() throws IOException, InterruptedException {
String path = "./testWriteFiles";
FileUtils.deleteDirectory(new File(path));
Field[] fields = new Field[2];
fields[0] = new Field("name", DataTypes.STRING);
fields[1] = new Field("age", DataTypes.INT);
TestUtil.writeFilesAndVerify(new Schema(fields), path, true);
CarbonReader reader = CarbonReader.builder(path, "_temp")
.projection(new String[]{"name", "age"}).build();
// expected output after sorting
String[] name = new String[100];
int[] age = new int[100];
for (int i = 0; i < 100; i++) {
name[i] = "robot" + (i / 10);
age[i] = (i % 10) * 10 + i / 10;
}
int i = 0;
while (reader.hasNext()) {
Object[] row = (Object[]) reader.readNextRow();
// Default sort column is applied for dimensions. So, need to validate
accordingly
Assert.assertEquals(name[i], row[0]);
Assert.assertEquals(age[i], row[1]);
i++;
}
Assert.assertEquals(i, 100);
reader.close();
// Read again
CarbonReader reader2 = CarbonReader
.builder(path, "_temp")
.projection(new String[]{"name", "age"})
.build();
i = 0;
while (reader2.hasNext()) {
Object[] row = (Object[]) reader2.readNextRow();
// Default sort column is applied for dimensions. So, need to validate
accordingly
Assert.assertEquals(name[i], row[0]);
Assert.assertEquals(age[i], row[1]);
i++;
}
Assert.assertEquals(i, 100);
reader2.close();
FileUtils.deleteDirectory(new File(path));
}
{code}
There are some error when build reader in the second tie with SDK carbonReader:
When run this case first time:
Error1
{code:java}
java.lang.StringIndexOutOfBoundsException: String index out of range: -1
at java.lang.String.substring(String.java:1967)
at
org.apache.carbondata.core.util.path.CarbonTablePath$DataFileUtil.getTaskNo(CarbonTablePath.java:510)
at
org.apache.carbondata.hadoop.api.CarbonInputFormat.getDataBlocksOfSegment(CarbonInputFormat.java:372)
at
org.apache.carbondata.hadoop.api.CarbonFileInputFormat.getSplits(CarbonFileInputFormat.java:197)
at
org.apache.carbondata.hadoop.api.CarbonFileInputFormat.getSplits(CarbonFileInputFormat.java:166)
at
org.apache.carbondata.sdk.file.CarbonReaderBuilder.build(CarbonReaderBuilder.java:160)
at
org.apache.carbondata.sdk.file.CarbonReaderTest.testWriteAndReadFiles(CarbonReaderTest.java:64)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at junit.framework.TestCase.runTest(TestCase.java:176)
at junit.framework.TestCase.runBare(TestCase.java:141)
at junit.framework.TestResult$1.protect(TestResult.java:122)
at junit.framework.TestResult.runProtected(TestResult.java:142)
at junit.framework.TestResult.run(TestResult.java:125)
at junit.framework.TestCase.run(TestCase.java:129)
at junit.framework.TestSuite.runTest(TestSuite.java:255)
at junit.framework.TestSuite.run(TestSuite.java:250)
at
org.junit.internal.runners.JUnit38ClassRunner.run(JUnit38ClassRunner.java:84)
at org.junit.runner.JUnitCore.run(JUnitCore.java:160)
at
com.intellij.junit4.JUnit4IdeaTestRunner.startRunnerWithArgs(JUnit4IdeaTestRunner.java:68)
at
com.intellij.rt.execution.junit.IdeaTestRunner$Repeater.startRunnerWithArgs(IdeaTestRunner.java:47)
at
com.intellij.rt.execution.junit.JUnitStarter.prepareStreamsAndStart(JUnitStarter.java:242)
at
com.intellij.rt.execution.junit.JUnitStarter.main(JUnitStarter.java:70)
{code}
sometimes has the second exception:
Error2:
{code:java}
java.lang.NegativeArraySizeException
at
org.apache.carbondata.core.indexstore.row.UnsafeDataMapRow.convertToSafeRow(UnsafeDataMapRow.java:234)
at
org.apache.carbondata.core.indexstore.row.UnsafeDataMapRow.convertToSafeRow(UnsafeDataMapRow.java:245)
at
org.apache.carbondata.core.indexstore.blockletindex.BlockletDataMap.prune(BlockletDataMap.java:648)
at
org.apache.carbondata.core.indexstore.blockletindex.BlockletDataMap.prune(BlockletDataMap.java:705)
at
org.apache.carbondata.core.indexstore.blockletindex.BlockletDataMapFactory.getAllBlocklets(BlockletDataMapFactory.java:297)
at
org.apache.carbondata.core.datamap.TableDataMap.prune(TableDataMap.java:92)
at
org.apache.carbondata.core.datamap.dev.expr.DataMapExprWrapperImpl.prune(DataMapExprWrapperImpl.java:52)
at
org.apache.carbondata.hadoop.api.CarbonInputFormat.getPrunedBlocklets(CarbonInputFormat.java:425)
at
org.apache.carbondata.hadoop.api.CarbonInputFormat.getDataBlocksOfSegment(CarbonInputFormat.java:362)
at
org.apache.carbondata.hadoop.api.CarbonFileInputFormat.getSplits(CarbonFileInputFormat.java:197)
at
org.apache.carbondata.hadoop.api.CarbonFileInputFormat.getSplits(CarbonFileInputFormat.java:166)
at
org.apache.carbondata.sdk.file.CarbonReaderBuilder.build(CarbonReaderBuilder.java:160)
at
org.apache.carbondata.sdk.file.CarbonReaderTest.testWriteAndReadFiles(CarbonReaderTest.java:64)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at junit.framework.TestCase.runTest(TestCase.java:176)
at junit.framework.TestCase.runBare(TestCase.java:141)
at junit.framework.TestResult$1.protect(TestResult.java:122)
at junit.framework.TestResult.runProtected(TestResult.java:142)
at junit.framework.TestResult.run(TestResult.java:125)
at junit.framework.TestCase.run(TestCase.java:129)
at junit.framework.TestSuite.runTest(TestSuite.java:255)
at junit.framework.TestSuite.run(TestSuite.java:250)
at
org.junit.internal.runners.JUnit38ClassRunner.run(JUnit38ClassRunner.java:84)
at org.junit.runner.JUnitCore.run(JUnitCore.java:160)
at
com.intellij.junit4.JUnit4IdeaTestRunner.startRunnerWithArgs(JUnit4IdeaTestRunner.java:68)
at
com.intellij.rt.execution.junit.IdeaTestRunner$Repeater.startRunnerWithArgs(IdeaTestRunner.java:47)
at
com.intellij.rt.execution.junit.JUnitStarter.prepareStreamsAndStart(JUnitStarter.java:242)
at
com.intellij.rt.execution.junit.JUnitStarter.main(JUnitStarter.java:70)
{code}
was:
Test code in org.apache.carbondata.sdk.file.CarbonReaderTest:
{code:java}
@Test
public void testWriteAndReadFiles() throws IOException, InterruptedException {
String path = "./testWriteFiles";
FileUtils.deleteDirectory(new File(path));
Field[] fields = new Field[2];
fields[0] = new Field("name", DataTypes.STRING);
fields[1] = new Field("age", DataTypes.INT);
TestUtil.writeFilesAndVerify(new Schema(fields), path, true);
CarbonReader reader = CarbonReader.builder(path, "_temp")
.projection(new String[]{"name", "age"}).build();
// expected output after sorting
String[] name = new String[100];
int[] age = new int[100];
for (int i = 0; i < 100; i++) {
name[i] = "robot" + (i / 10);
age[i] = (i % 10) * 10 + i / 10;
}
int i = 0;
while (reader.hasNext()) {
Object[] row = (Object[]) reader.readNextRow();
// Default sort column is applied for dimensions. So, need to validate
accordingly
Assert.assertEquals(name[i], row[0]);
Assert.assertEquals(age[i], row[1]);
i++;
}
Assert.assertEquals(i, 100);
reader.close();
// Read again
CarbonReader reader2 = CarbonReader
.builder(path, "_temp")
.projection(new String[]{"name", "age"})
.build();
i = 0;
while (reader2.hasNext()) {
Object[] row = (Object[]) reader2.readNextRow();
// Default sort column is applied for dimensions. So, need to validate
accordingly
Assert.assertEquals(name[i], row[0]);
Assert.assertEquals(age[i], row[1]);
i++;
}
Assert.assertEquals(i, 100);
reader2.close();
FileUtils.deleteDirectory(new File(path));
}
{code}
When run the test case, it will throw some exception:
When run this case first time:
Error1
{code:java}
java.lang.StringIndexOutOfBoundsException: String index out of range: -1
at java.lang.String.substring(String.java:1967)
at
org.apache.carbondata.core.util.path.CarbonTablePath$DataFileUtil.getTaskNo(CarbonTablePath.java:510)
at
org.apache.carbondata.hadoop.api.CarbonInputFormat.getDataBlocksOfSegment(CarbonInputFormat.java:372)
at
org.apache.carbondata.hadoop.api.CarbonFileInputFormat.getSplits(CarbonFileInputFormat.java:197)
at
org.apache.carbondata.hadoop.api.CarbonFileInputFormat.getSplits(CarbonFileInputFormat.java:166)
at
org.apache.carbondata.sdk.file.CarbonReaderBuilder.build(CarbonReaderBuilder.java:160)
at
org.apache.carbondata.sdk.file.CarbonReaderTest.testWriteAndReadFiles(CarbonReaderTest.java:64)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at junit.framework.TestCase.runTest(TestCase.java:176)
at junit.framework.TestCase.runBare(TestCase.java:141)
at junit.framework.TestResult$1.protect(TestResult.java:122)
at junit.framework.TestResult.runProtected(TestResult.java:142)
at junit.framework.TestResult.run(TestResult.java:125)
at junit.framework.TestCase.run(TestCase.java:129)
at junit.framework.TestSuite.runTest(TestSuite.java:255)
at junit.framework.TestSuite.run(TestSuite.java:250)
at
org.junit.internal.runners.JUnit38ClassRunner.run(JUnit38ClassRunner.java:84)
at org.junit.runner.JUnitCore.run(JUnitCore.java:160)
at
com.intellij.junit4.JUnit4IdeaTestRunner.startRunnerWithArgs(JUnit4IdeaTestRunner.java:68)
at
com.intellij.rt.execution.junit.IdeaTestRunner$Repeater.startRunnerWithArgs(IdeaTestRunner.java:47)
at
com.intellij.rt.execution.junit.JUnitStarter.prepareStreamsAndStart(JUnitStarter.java:242)
at
com.intellij.rt.execution.junit.JUnitStarter.main(JUnitStarter.java:70)
{code}
sometimes has the second exception:
Error2:
{code:java}
java.lang.NegativeArraySizeException
at
org.apache.carbondata.core.indexstore.row.UnsafeDataMapRow.convertToSafeRow(UnsafeDataMapRow.java:234)
at
org.apache.carbondata.core.indexstore.row.UnsafeDataMapRow.convertToSafeRow(UnsafeDataMapRow.java:245)
at
org.apache.carbondata.core.indexstore.blockletindex.BlockletDataMap.prune(BlockletDataMap.java:648)
at
org.apache.carbondata.core.indexstore.blockletindex.BlockletDataMap.prune(BlockletDataMap.java:705)
at
org.apache.carbondata.core.indexstore.blockletindex.BlockletDataMapFactory.getAllBlocklets(BlockletDataMapFactory.java:297)
at
org.apache.carbondata.core.datamap.TableDataMap.prune(TableDataMap.java:92)
at
org.apache.carbondata.core.datamap.dev.expr.DataMapExprWrapperImpl.prune(DataMapExprWrapperImpl.java:52)
at
org.apache.carbondata.hadoop.api.CarbonInputFormat.getPrunedBlocklets(CarbonInputFormat.java:425)
at
org.apache.carbondata.hadoop.api.CarbonInputFormat.getDataBlocksOfSegment(CarbonInputFormat.java:362)
at
org.apache.carbondata.hadoop.api.CarbonFileInputFormat.getSplits(CarbonFileInputFormat.java:197)
at
org.apache.carbondata.hadoop.api.CarbonFileInputFormat.getSplits(CarbonFileInputFormat.java:166)
at
org.apache.carbondata.sdk.file.CarbonReaderBuilder.build(CarbonReaderBuilder.java:160)
at
org.apache.carbondata.sdk.file.CarbonReaderTest.testWriteAndReadFiles(CarbonReaderTest.java:64)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at junit.framework.TestCase.runTest(TestCase.java:176)
at junit.framework.TestCase.runBare(TestCase.java:141)
at junit.framework.TestResult$1.protect(TestResult.java:122)
at junit.framework.TestResult.runProtected(TestResult.java:142)
at junit.framework.TestResult.run(TestResult.java:125)
at junit.framework.TestCase.run(TestCase.java:129)
at junit.framework.TestSuite.runTest(TestSuite.java:255)
at junit.framework.TestSuite.run(TestSuite.java:250)
at
org.junit.internal.runners.JUnit38ClassRunner.run(JUnit38ClassRunner.java:84)
at org.junit.runner.JUnitCore.run(JUnitCore.java:160)
at
com.intellij.junit4.JUnit4IdeaTestRunner.startRunnerWithArgs(JUnit4IdeaTestRunner.java:68)
at
com.intellij.rt.execution.junit.IdeaTestRunner$Repeater.startRunnerWithArgs(IdeaTestRunner.java:47)
at
com.intellij.rt.execution.junit.JUnitStarter.prepareStreamsAndStart(JUnitStarter.java:242)
at
com.intellij.rt.execution.junit.JUnitStarter.main(JUnitStarter.java:70)
{code}
> There are some error when reader twice with SDK carbonReader
> ------------------------------------------------------------
>
> Key: CARBONDATA-2491
> URL: https://issues.apache.org/jira/browse/CARBONDATA-2491
> Project: CarbonData
> Issue Type: Bug
> Reporter: xubo245
> Assignee: xubo245
> Priority: Major
> Time Spent: 2h 50m
> Remaining Estimate: 0h
>
> Test code in org.apache.carbondata.sdk.file.CarbonReaderTest:
> {code:java}
> @Test
> public void testWriteAndReadFiles() throws IOException,
> InterruptedException {
> String path = "./testWriteFiles";
> FileUtils.deleteDirectory(new File(path));
> Field[] fields = new Field[2];
> fields[0] = new Field("name", DataTypes.STRING);
> fields[1] = new Field("age", DataTypes.INT);
> TestUtil.writeFilesAndVerify(new Schema(fields), path, true);
> CarbonReader reader = CarbonReader.builder(path, "_temp")
> .projection(new String[]{"name", "age"}).build();
> // expected output after sorting
> String[] name = new String[100];
> int[] age = new int[100];
> for (int i = 0; i < 100; i++) {
> name[i] = "robot" + (i / 10);
> age[i] = (i % 10) * 10 + i / 10;
> }
> int i = 0;
> while (reader.hasNext()) {
> Object[] row = (Object[]) reader.readNextRow();
> // Default sort column is applied for dimensions. So, need to validate
> accordingly
> Assert.assertEquals(name[i], row[0]);
> Assert.assertEquals(age[i], row[1]);
> i++;
> }
> Assert.assertEquals(i, 100);
> reader.close();
> // Read again
> CarbonReader reader2 = CarbonReader
> .builder(path, "_temp")
> .projection(new String[]{"name", "age"})
> .build();
> i = 0;
> while (reader2.hasNext()) {
> Object[] row = (Object[]) reader2.readNextRow();
> // Default sort column is applied for dimensions. So, need to validate
> accordingly
> Assert.assertEquals(name[i], row[0]);
> Assert.assertEquals(age[i], row[1]);
> i++;
> }
> Assert.assertEquals(i, 100);
> reader2.close();
> FileUtils.deleteDirectory(new File(path));
> }
> {code}
> There are some error when build reader in the second tie with SDK
> carbonReader:
> When run this case first time:
> Error1
> {code:java}
> java.lang.StringIndexOutOfBoundsException: String index out of range: -1
> at java.lang.String.substring(String.java:1967)
> at
> org.apache.carbondata.core.util.path.CarbonTablePath$DataFileUtil.getTaskNo(CarbonTablePath.java:510)
> at
> org.apache.carbondata.hadoop.api.CarbonInputFormat.getDataBlocksOfSegment(CarbonInputFormat.java:372)
> at
> org.apache.carbondata.hadoop.api.CarbonFileInputFormat.getSplits(CarbonFileInputFormat.java:197)
> at
> org.apache.carbondata.hadoop.api.CarbonFileInputFormat.getSplits(CarbonFileInputFormat.java:166)
> at
> org.apache.carbondata.sdk.file.CarbonReaderBuilder.build(CarbonReaderBuilder.java:160)
> at
> org.apache.carbondata.sdk.file.CarbonReaderTest.testWriteAndReadFiles(CarbonReaderTest.java:64)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:498)
> at junit.framework.TestCase.runTest(TestCase.java:176)
> at junit.framework.TestCase.runBare(TestCase.java:141)
> at junit.framework.TestResult$1.protect(TestResult.java:122)
> at junit.framework.TestResult.runProtected(TestResult.java:142)
> at junit.framework.TestResult.run(TestResult.java:125)
> at junit.framework.TestCase.run(TestCase.java:129)
> at junit.framework.TestSuite.runTest(TestSuite.java:255)
> at junit.framework.TestSuite.run(TestSuite.java:250)
> at
> org.junit.internal.runners.JUnit38ClassRunner.run(JUnit38ClassRunner.java:84)
> at org.junit.runner.JUnitCore.run(JUnitCore.java:160)
> at
> com.intellij.junit4.JUnit4IdeaTestRunner.startRunnerWithArgs(JUnit4IdeaTestRunner.java:68)
> at
> com.intellij.rt.execution.junit.IdeaTestRunner$Repeater.startRunnerWithArgs(IdeaTestRunner.java:47)
> at
> com.intellij.rt.execution.junit.JUnitStarter.prepareStreamsAndStart(JUnitStarter.java:242)
> at
> com.intellij.rt.execution.junit.JUnitStarter.main(JUnitStarter.java:70)
> {code}
> sometimes has the second exception:
> Error2:
> {code:java}
> java.lang.NegativeArraySizeException
> at
> org.apache.carbondata.core.indexstore.row.UnsafeDataMapRow.convertToSafeRow(UnsafeDataMapRow.java:234)
> at
> org.apache.carbondata.core.indexstore.row.UnsafeDataMapRow.convertToSafeRow(UnsafeDataMapRow.java:245)
> at
> org.apache.carbondata.core.indexstore.blockletindex.BlockletDataMap.prune(BlockletDataMap.java:648)
> at
> org.apache.carbondata.core.indexstore.blockletindex.BlockletDataMap.prune(BlockletDataMap.java:705)
> at
> org.apache.carbondata.core.indexstore.blockletindex.BlockletDataMapFactory.getAllBlocklets(BlockletDataMapFactory.java:297)
> at
> org.apache.carbondata.core.datamap.TableDataMap.prune(TableDataMap.java:92)
> at
> org.apache.carbondata.core.datamap.dev.expr.DataMapExprWrapperImpl.prune(DataMapExprWrapperImpl.java:52)
> at
> org.apache.carbondata.hadoop.api.CarbonInputFormat.getPrunedBlocklets(CarbonInputFormat.java:425)
> at
> org.apache.carbondata.hadoop.api.CarbonInputFormat.getDataBlocksOfSegment(CarbonInputFormat.java:362)
> at
> org.apache.carbondata.hadoop.api.CarbonFileInputFormat.getSplits(CarbonFileInputFormat.java:197)
> at
> org.apache.carbondata.hadoop.api.CarbonFileInputFormat.getSplits(CarbonFileInputFormat.java:166)
> at
> org.apache.carbondata.sdk.file.CarbonReaderBuilder.build(CarbonReaderBuilder.java:160)
> at
> org.apache.carbondata.sdk.file.CarbonReaderTest.testWriteAndReadFiles(CarbonReaderTest.java:64)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:498)
> at junit.framework.TestCase.runTest(TestCase.java:176)
> at junit.framework.TestCase.runBare(TestCase.java:141)
> at junit.framework.TestResult$1.protect(TestResult.java:122)
> at junit.framework.TestResult.runProtected(TestResult.java:142)
> at junit.framework.TestResult.run(TestResult.java:125)
> at junit.framework.TestCase.run(TestCase.java:129)
> at junit.framework.TestSuite.runTest(TestSuite.java:255)
> at junit.framework.TestSuite.run(TestSuite.java:250)
> at
> org.junit.internal.runners.JUnit38ClassRunner.run(JUnit38ClassRunner.java:84)
> at org.junit.runner.JUnitCore.run(JUnitCore.java:160)
> at
> com.intellij.junit4.JUnit4IdeaTestRunner.startRunnerWithArgs(JUnit4IdeaTestRunner.java:68)
> at
> com.intellij.rt.execution.junit.IdeaTestRunner$Repeater.startRunnerWithArgs(IdeaTestRunner.java:47)
> at
> com.intellij.rt.execution.junit.JUnitStarter.prepareStreamsAndStart(JUnitStarter.java:242)
> at
> com.intellij.rt.execution.junit.JUnitStarter.main(JUnitStarter.java:70)
> {code}
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
