Repository: carbondata Updated Branches: refs/heads/spark-2.3 8fe165668 -> 041603dcc
[CARBONDATA-2413] After running CarbonWriter, there is null/_system directory about datamap After running CarbonWriter, there is null directory: ***null/_system# ls datamap.mdtfile **# git status Fix: Don't create data map file/directory when using SDK. According to: 1.segnemt id is null 2.add carbon properties for SDK This closes #2246 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/cf55028f Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/cf55028f Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/cf55028f Branch: refs/heads/spark-2.3 Commit: cf55028f7b5471e8bb6242c10b06129013eb1951 Parents: b8d5abf Author: xubo245 <xub...@huawei.com> Authored: Thu May 17 12:13:01 2018 +0800 Committer: ravipesala <ravi.pes...@gmail.com> Committed: Sat May 19 08:14:17 2018 +0530 ---------------------------------------------------------------------- .../chunk/reader/MeasureColumnChunkReader.java | 2 +- .../table/DiskBasedDMSchemaStorageProvider.java | 14 ++++--- .../carbondata/core/scan/filter/FilterUtil.java | 2 +- .../scan/result/iterator/RawResultIterator.java | 4 +- .../management/CarbonLoadDataCommand.scala | 2 +- .../datasources/SparkCarbonTableFormat.scala | 2 +- .../loading/DataLoadProcessBuilder.java | 2 +- .../loading/model/CarbonLoadModel.java | 14 +++---- .../sdk/file/CarbonWriterBuilder.java | 2 +- .../sdk/file/AvroCarbonWriterTest.java | 13 +++++- .../sdk/file/CSVCarbonWriterTest.java | 12 ++++++ .../CSVNonTransactionalCarbonWriterTest.java | 11 +++++ .../carbondata/sdk/file/CarbonReaderTest.java | 12 ++++++ .../apache/carbondata/sdk/file/TestUtil.java | 44 ++++++++++++++++++++ .../carbondata/store/LocalCarbonStoreTest.java | 11 +++++ 15 files changed, 125 insertions(+), 22 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/carbondata/blob/cf55028f/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/MeasureColumnChunkReader.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/MeasureColumnChunkReader.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/MeasureColumnChunkReader.java index bf76025..f1392d0 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/MeasureColumnChunkReader.java +++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/MeasureColumnChunkReader.java @@ -49,7 +49,7 @@ public interface MeasureColumnChunkReader { throws IOException; /** - * Covert raw data to measure chunk + * Convert raw data to measure chunk * @param measureRawColumnChunk * @param pageNumber * @return http://git-wip-us.apache.org/repos/asf/carbondata/blob/cf55028f/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/DiskBasedDMSchemaStorageProvider.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/DiskBasedDMSchemaStorageProvider.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/DiskBasedDMSchemaStorageProvider.java index 4ebbcd0..cf4f6b9 100644 --- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/DiskBasedDMSchemaStorageProvider.java +++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/DiskBasedDMSchemaStorageProvider.java @@ -84,7 +84,7 @@ public class DiskBasedDMSchemaStorageProvider implements DataMapSchemaStoragePro if (null != brWriter) { brWriter.flush(); } - checkAndReloadDataMapSchemas(); + checkAndReloadDataMapSchemas(true); dataMapSchemas.add(dataMapSchema); touchMDTFile(); CarbonUtil.closeStreams(dataOutputStream, brWriter); @@ -93,7 +93,7 @@ public class DiskBasedDMSchemaStorageProvider implements DataMapSchemaStoragePro @Override public DataMapSchema retrieveSchema(String dataMapName) throws IOException, NoSuchDataMapException { - checkAndReloadDataMapSchemas(); + checkAndReloadDataMapSchemas(true); for (DataMapSchema dataMapSchema : dataMapSchemas) { if (dataMapSchema.getDataMapName().equalsIgnoreCase(dataMapName)) { return dataMapSchema; @@ -103,7 +103,7 @@ public class DiskBasedDMSchemaStorageProvider implements DataMapSchemaStoragePro } @Override public List<DataMapSchema> retrieveSchemas(CarbonTable carbonTable) throws IOException { - checkAndReloadDataMapSchemas(); + checkAndReloadDataMapSchemas(false); List<DataMapSchema> dataMapSchemas = new ArrayList<>(); for (DataMapSchema dataMapSchema : this.dataMapSchemas) { List<RelationIdentifier> parentTables = dataMapSchema.getParentTables(); @@ -119,7 +119,7 @@ public class DiskBasedDMSchemaStorageProvider implements DataMapSchemaStoragePro } @Override public List<DataMapSchema> retrieveAllSchemas() throws IOException { - checkAndReloadDataMapSchemas(); + checkAndReloadDataMapSchemas(true); return new ArrayList<>(dataMapSchemas); } @@ -175,7 +175,7 @@ public class DiskBasedDMSchemaStorageProvider implements DataMapSchemaStoragePro } } - private void checkAndReloadDataMapSchemas() throws IOException { + private void checkAndReloadDataMapSchemas(boolean touchFile) throws IOException { if (FileFactory.isFileExist(mdtFilePath)) { long lastModifiedTime = FileFactory.getCarbonFile(mdtFilePath).getLastModifiedTime(); if (this.lastModifiedTime != lastModifiedTime) { @@ -183,7 +183,9 @@ public class DiskBasedDMSchemaStorageProvider implements DataMapSchemaStoragePro this.lastModifiedTime = lastModifiedTime; } } else { - touchMDTFile(); + if (touchFile) { + touchMDTFile(); + } dataMapSchemas = retrieveAllSchemasInternal(); } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/cf55028f/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java index 2032ddb..5196f8f 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java @@ -885,7 +885,7 @@ public final class FilterUtil { } /** - * Below method will be used to covert the filter surrogate keys + * Below method will be used to convert the filter surrogate keys * to mdkey * * @param columnFilterInfo http://git-wip-us.apache.org/repos/asf/carbondata/blob/cf55028f/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/RawResultIterator.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/RawResultIterator.java b/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/RawResultIterator.java index 1fe50a2..94cea91 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/RawResultIterator.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/RawResultIterator.java @@ -191,9 +191,9 @@ public class RawResultIterator extends CarbonIterator<Object[]> { private Object[] convertRow(Object[] rawRow) throws KeyGenException { byte[] dims = ((ByteArrayWrapper) rawRow[0]).getDictionaryKey(); long[] keyArray = sourceSegProperties.getDimensionKeyGenerator().getKeyArray(dims); - byte[] covertedBytes = + byte[] convertedBytes = destinationSegProperties.getDimensionKeyGenerator().generateKey(keyArray); - ((ByteArrayWrapper) rawRow[0]).setDictionaryKey(covertedBytes); + ((ByteArrayWrapper) rawRow[0]).setDictionaryKey(convertedBytes); return rawRow; } http://git-wip-us.apache.org/repos/asf/carbondata/blob/cf55028f/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala ---------------------------------------------------------------------- diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala index 5ce510b..ba062c0 100644 --- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala +++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala @@ -858,7 +858,7 @@ case class CarbonLoadDataCommand( } /** - * Convert the rdd as per steps of data loading inputprocessor step and coverter step + * Convert the rdd as per steps of data loading inputprocessor step and converter step * @param originRDD * @param sparkSession * @param model http://git-wip-us.apache.org/repos/asf/carbondata/blob/cf55028f/integration/spark2/src/main/scala/org/apache/spark/sql/execution/datasources/SparkCarbonTableFormat.scala ---------------------------------------------------------------------- diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/datasources/SparkCarbonTableFormat.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/datasources/SparkCarbonTableFormat.scala index ac41d2e..42f1f77 100644 --- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/datasources/SparkCarbonTableFormat.scala +++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/datasources/SparkCarbonTableFormat.scala @@ -110,7 +110,7 @@ with Serializable { model.setDictionaryServerHost(options.getOrElse("dicthost", null)) model.setDictionaryServerPort(options.getOrElse("dictport", "-1").toInt) CarbonTableOutputFormat.setOverwrite(conf, options("overwrite").toBoolean) - model.setLoadWithoutCoverterStep(true) + model.setLoadWithoutConverterStep(true) val staticPartition = options.getOrElse("staticpartition", null) if (staticPartition != null) { http://git-wip-us.apache.org/repos/asf/carbondata/blob/cf55028f/processing/src/main/java/org/apache/carbondata/processing/loading/DataLoadProcessBuilder.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/DataLoadProcessBuilder.java b/processing/src/main/java/org/apache/carbondata/processing/loading/DataLoadProcessBuilder.java index 17d0c76..b7b725c 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/loading/DataLoadProcessBuilder.java +++ b/processing/src/main/java/org/apache/carbondata/processing/loading/DataLoadProcessBuilder.java @@ -62,7 +62,7 @@ public final class DataLoadProcessBuilder { CarbonIterator[] inputIterators) throws Exception { CarbonDataLoadConfiguration configuration = createConfiguration(loadModel, storeLocation); SortScopeOptions.SortScope sortScope = CarbonDataProcessorUtil.getSortScope(configuration); - if (loadModel.isLoadWithoutCoverterStep()) { + if (loadModel.isLoadWithoutConverterStep()) { return buildInternalWithNoConverter(inputIterators, configuration, sortScope); } else if (!configuration.isSortTable() || sortScope.equals(SortScopeOptions.SortScope.NO_SORT)) { http://git-wip-us.apache.org/repos/asf/carbondata/blob/cf55028f/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModel.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModel.java b/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModel.java index 0cc0da3..f82de83 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModel.java +++ b/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModel.java @@ -111,7 +111,7 @@ public class CarbonLoadModel implements Serializable { private String defaultDateFormat; /** - * defines the string that should be treated as null while loadind data + * defines the string that should be treated as null while loading data */ private String serializationNullFormat; @@ -204,7 +204,7 @@ public class CarbonLoadModel implements Serializable { * For this method there will be no data conversion step. It writes data which is directly * pushed into. */ - private boolean isLoadWithoutCoverterStep; + private boolean isLoadWithoutConverterStep; /** * Flder path to where data should be written for this load. @@ -437,7 +437,7 @@ public class CarbonLoadModel implements Serializable { copy.batchSortSizeInMb = batchSortSizeInMb; copy.isAggLoadRequest = isAggLoadRequest; copy.badRecordsLocation = badRecordsLocation; - copy.isLoadWithoutCoverterStep = isLoadWithoutCoverterStep; + copy.isLoadWithoutConverterStep = isLoadWithoutConverterStep; copy.sortColumnsBoundsStr = sortColumnsBoundsStr; return copy; } @@ -816,12 +816,12 @@ public class CarbonLoadModel implements Serializable { } - public boolean isLoadWithoutCoverterStep() { - return isLoadWithoutCoverterStep; + public boolean isLoadWithoutConverterStep() { + return isLoadWithoutConverterStep; } - public void setLoadWithoutCoverterStep(boolean loadWithoutCoverterStep) { - isLoadWithoutCoverterStep = loadWithoutCoverterStep; + public void setLoadWithoutConverterStep(boolean loadWithoutConverterStep) { + isLoadWithoutConverterStep = loadWithoutConverterStep; } public String getDataWritePath() { http://git-wip-us.apache.org/repos/asf/carbondata/blob/cf55028f/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonWriterBuilder.java ---------------------------------------------------------------------- diff --git a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonWriterBuilder.java b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonWriterBuilder.java index 0f9c9d7..585975f 100644 --- a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonWriterBuilder.java +++ b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonWriterBuilder.java @@ -336,7 +336,7 @@ public class CarbonWriterBuilder { // handle multi level complex type support. As there are no conversion converter step is // removed from the load. LoadWithoutConverter flag is going to point to the Loader Builder // which will skip Conversion Step. - loadModel.setLoadWithoutCoverterStep(true); + loadModel.setLoadWithoutConverterStep(true); return new AvroCarbonWriter(loadModel); } http://git-wip-us.apache.org/repos/asf/carbondata/blob/cf55028f/store/sdk/src/test/java/org/apache/carbondata/sdk/file/AvroCarbonWriterTest.java ---------------------------------------------------------------------- diff --git a/store/sdk/src/test/java/org/apache/carbondata/sdk/file/AvroCarbonWriterTest.java b/store/sdk/src/test/java/org/apache/carbondata/sdk/file/AvroCarbonWriterTest.java index 163512a..104c6e4 100644 --- a/store/sdk/src/test/java/org/apache/carbondata/sdk/file/AvroCarbonWriterTest.java +++ b/store/sdk/src/test/java/org/apache/carbondata/sdk/file/AvroCarbonWriterTest.java @@ -29,16 +29,27 @@ import org.apache.carbondata.core.util.path.CarbonTablePath; import org.apache.avro.generic.GenericData; import org.apache.commons.io.FileUtils; import org.apache.commons.lang.CharEncoding; +import org.junit.After; import org.junit.Assert; +import org.junit.Before; import org.junit.Test; import tech.allegro.schema.json2avro.converter.JsonAvroConverter; import org.apache.avro.Schema; - public class AvroCarbonWriterTest { private String path = "./AvroCarbonWriterSuiteWriteFiles"; + @Before + public void cleanFile() { + assert (TestUtil.cleanMdtFile()); + } + + @After + public void verifyDMFile() { + assert (!TestUtil.verifyMdtFile()); + } + @Test public void testWriteBasic() throws IOException { FileUtils.deleteDirectory(new File(path)); http://git-wip-us.apache.org/repos/asf/carbondata/blob/cf55028f/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CSVCarbonWriterTest.java ---------------------------------------------------------------------- diff --git a/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CSVCarbonWriterTest.java b/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CSVCarbonWriterTest.java index d68d85b..fc283b6 100644 --- a/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CSVCarbonWriterTest.java +++ b/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CSVCarbonWriterTest.java @@ -27,7 +27,9 @@ import org.apache.carbondata.core.metadata.datatype.DataTypes; import org.apache.carbondata.core.util.path.CarbonTablePath; import org.apache.commons.io.FileUtils; +import org.junit.After; import org.junit.Assert; +import org.junit.Before; import org.junit.Test; /** @@ -35,6 +37,16 @@ import org.junit.Test; */ public class CSVCarbonWriterTest { + @Before + public void cleanFile() { + assert (TestUtil.cleanMdtFile()); + } + + @After + public void verifyDMFile() { + assert (!TestUtil.verifyMdtFile()); + } + @Test public void testWriteFiles() throws IOException { String path = "./testWriteFiles"; http://git-wip-us.apache.org/repos/asf/carbondata/blob/cf55028f/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CSVNonTransactionalCarbonWriterTest.java ---------------------------------------------------------------------- diff --git a/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CSVNonTransactionalCarbonWriterTest.java b/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CSVNonTransactionalCarbonWriterTest.java index 19b0a42..881b5a5 100644 --- a/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CSVNonTransactionalCarbonWriterTest.java +++ b/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CSVNonTransactionalCarbonWriterTest.java @@ -27,13 +27,24 @@ import org.apache.carbondata.core.metadata.datatype.DataTypes; import org.apache.carbondata.core.util.path.CarbonTablePath; import org.apache.commons.io.FileUtils; +import org.junit.After; import org.junit.Assert; +import org.junit.Before; import org.junit.Test; /** * Test suite for {@link CSVCarbonWriter} */ public class CSVNonTransactionalCarbonWriterTest { + @Before + public void cleanFile() { + assert (TestUtil.cleanMdtFile()); + } + + @After + public void verifyDMFile() { + assert (!TestUtil.verifyMdtFile()); + } @Test public void testWriteFiles() throws IOException { http://git-wip-us.apache.org/repos/asf/carbondata/blob/cf55028f/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonReaderTest.java ---------------------------------------------------------------------- diff --git a/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonReaderTest.java b/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonReaderTest.java index f2c6d45..937dde8 100644 --- a/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonReaderTest.java +++ b/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonReaderTest.java @@ -27,11 +27,23 @@ import org.apache.carbondata.core.metadata.schema.table.TableInfo; import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema; import org.apache.commons.io.FileUtils; +import org.junit.After; import org.junit.Assert; +import org.junit.Before; import org.junit.Test; public class CarbonReaderTest { + @Before + public void cleanFile() { + assert (TestUtil.cleanMdtFile()); + } + + @After + public void verifyDMFile() { + assert (!TestUtil.verifyMdtFile()); + } + @Test public void testWriteAndReadFiles() throws IOException, InterruptedException { String path = "./testWriteFiles"; http://git-wip-us.apache.org/repos/asf/carbondata/blob/cf55028f/store/sdk/src/test/java/org/apache/carbondata/sdk/file/TestUtil.java ---------------------------------------------------------------------- diff --git a/store/sdk/src/test/java/org/apache/carbondata/sdk/file/TestUtil.java b/store/sdk/src/test/java/org/apache/carbondata/sdk/file/TestUtil.java index 6870f36..97de1a0 100644 --- a/store/sdk/src/test/java/org/apache/carbondata/sdk/file/TestUtil.java +++ b/store/sdk/src/test/java/org/apache/carbondata/sdk/file/TestUtil.java @@ -23,6 +23,8 @@ import java.io.IOException; import org.apache.carbondata.common.exceptions.sql.InvalidLoadOptionException; import org.apache.carbondata.core.constants.CarbonCommonConstants; +import org.apache.carbondata.core.datastore.impl.FileFactory; +import org.apache.carbondata.core.util.CarbonProperties; import org.apache.carbondata.core.util.path.CarbonTablePath; import org.junit.Assert; @@ -108,4 +110,46 @@ public class TestUtil { Assert.assertNotNull(dataFiles); Assert.assertTrue(dataFiles.length > 0); } + + /** + * verify whether the file exists + * if delete the file success or file not exists, then return true; otherwise return false + * + * @return boolean + */ + public static boolean cleanMdtFile() { + String fileName = CarbonProperties.getInstance().getSystemFolderLocation() + + CarbonCommonConstants.FILE_SEPARATOR + "datamap.mdtfile"; + try { + if (FileFactory.isFileExist(fileName)) { + File file = new File(fileName); + file.delete(); + return true; + } else { + return true; + } + } catch (IOException e) { + e.printStackTrace(); + return false; + } + } + + /** + * verify whether the mdt file exists + * if the file exists, then return true; otherwise return false + * + * @return boolean + */ + public static boolean verifyMdtFile() { + String fileName = CarbonProperties.getInstance().getSystemFolderLocation() + + CarbonCommonConstants.FILE_SEPARATOR + "datamap.mdtfile"; + try { + if (FileFactory.isFileExist(fileName)) { + return true; + } + return false; + } catch (IOException e) { + throw new RuntimeException("IO exception:", e); + } + } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/cf55028f/store/sdk/src/test/java/org/apache/carbondata/store/LocalCarbonStoreTest.java ---------------------------------------------------------------------- diff --git a/store/sdk/src/test/java/org/apache/carbondata/store/LocalCarbonStoreTest.java b/store/sdk/src/test/java/org/apache/carbondata/store/LocalCarbonStoreTest.java index a5b5edc..51d0b27 100644 --- a/store/sdk/src/test/java/org/apache/carbondata/store/LocalCarbonStoreTest.java +++ b/store/sdk/src/test/java/org/apache/carbondata/store/LocalCarbonStoreTest.java @@ -28,9 +28,20 @@ import org.apache.carbondata.sdk.file.Schema; import org.apache.carbondata.sdk.file.TestUtil; import org.apache.commons.io.FileUtils; +import org.junit.After; +import org.junit.Before; import org.junit.Test; public class LocalCarbonStoreTest { + @Before + public void cleanFile() { + assert (TestUtil.cleanMdtFile()); + } + + @After + public void verifyDMFile() { + assert (!TestUtil.verifyMdtFile()); + } // TODO: complete this testcase // Currently result rows are empty, because SDK is not writing table status file