http://git-wip-us.apache.org/repos/asf/carbondata/blob/5663e916/processing/src/main/java/org/apache/carbondata/processing/loading/steps/DataWriterBatchProcessorStepImpl.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/steps/DataWriterBatchProcessorStepImpl.java b/processing/src/main/java/org/apache/carbondata/processing/loading/steps/DataWriterBatchProcessorStepImpl.java index f030d52..369c1f2 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/loading/steps/DataWriterBatchProcessorStepImpl.java +++ b/processing/src/main/java/org/apache/carbondata/processing/loading/steps/DataWriterBatchProcessorStepImpl.java @@ -24,6 +24,7 @@ import org.apache.carbondata.common.logging.LogServiceFactory; import org.apache.carbondata.core.datastore.row.CarbonRow; import org.apache.carbondata.core.metadata.CarbonTableIdentifier; import org.apache.carbondata.core.util.CarbonTimeStatisticsFactory; +import org.apache.carbondata.core.util.path.CarbonTablePath; import org.apache.carbondata.processing.loading.AbstractDataLoadProcessorStep; import org.apache.carbondata.processing.loading.CarbonDataLoadConfiguration; import org.apache.carbondata.processing.loading.DataField; @@ -59,13 +60,11 @@ public class DataWriterBatchProcessorStepImpl extends AbstractDataLoadProcessorS child.initialize(); } - private String[] getStoreLocation(CarbonTableIdentifier tableIdentifier, String partitionId) { - String[] storeLocation = CarbonDataProcessorUtil - .getLocalDataFolderLocation(tableIdentifier.getDatabaseName(), - tableIdentifier.getTableName(), String.valueOf(configuration.getTaskNo()), partitionId, - configuration.getSegmentId() + "", false, false); - CarbonDataProcessorUtil.createLocations(storeLocation); - return storeLocation; + private String[] getStoreLocation(CarbonTableIdentifier tableIdentifier) { + return CarbonDataProcessorUtil.getLocalDataFolderLocation( + tableIdentifier.getDatabaseName(), tableIdentifier.getTableName(), + String.valueOf(configuration.getTaskNo()), + configuration.getSegmentId(), false, false); } @Override public Iterator<CarbonRowBatch>[] execute() throws CarbonDataLoadingException { @@ -75,18 +74,19 @@ public class DataWriterBatchProcessorStepImpl extends AbstractDataLoadProcessorS String tableName = tableIdentifier.getTableName(); try { CarbonTimeStatisticsFactory.getLoadStatisticsInstance() - .recordDictionaryValue2MdkAdd2FileTime(configuration.getPartitionId(), + .recordDictionaryValue2MdkAdd2FileTime(CarbonTablePath.DEPRECATED_PATITION_ID, System.currentTimeMillis()); int i = 0; + String[] storeLocation = getStoreLocation(tableIdentifier); + CarbonDataProcessorUtil.createLocations(storeLocation); for (Iterator<CarbonRowBatch> iterator : iterators) { - String[] storeLocation = getStoreLocation(tableIdentifier, String.valueOf(i)); int k = 0; while (iterator.hasNext()) { CarbonRowBatch next = iterator.next(); // If no rows from merge sorter, then don't create a file in fact column handler if (next.hasNext()) { CarbonFactDataHandlerModel model = CarbonFactDataHandlerModel - .createCarbonFactDataHandlerModel(configuration, storeLocation, i, k++); + .createCarbonFactDataHandlerModel(configuration, storeLocation, 0, k++); CarbonFactHandler dataHandler = CarbonFactHandlerFactory .createCarbonFactHandler(model, CarbonFactHandlerFactory.FactHandlerType.COLUMNAR); dataHandler.initialise(); @@ -119,10 +119,11 @@ public class DataWriterBatchProcessorStepImpl extends AbstractDataLoadProcessorS CarbonTimeStatisticsFactory.getLoadStatisticsInstance().recordTotalRecords(rowCounter.get()); processingComplete(dataHandler); CarbonTimeStatisticsFactory.getLoadStatisticsInstance() - .recordDictionaryValue2MdkAdd2FileTime(configuration.getPartitionId(), + .recordDictionaryValue2MdkAdd2FileTime(CarbonTablePath.DEPRECATED_PATITION_ID, System.currentTimeMillis()); CarbonTimeStatisticsFactory.getLoadStatisticsInstance() - .recordMdkGenerateTotalTime(configuration.getPartitionId(), System.currentTimeMillis()); + .recordMdkGenerateTotalTime(CarbonTablePath.DEPRECATED_PATITION_ID, + System.currentTimeMillis()); } private void processingComplete(CarbonFactHandler dataHandler) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5663e916/processing/src/main/java/org/apache/carbondata/processing/loading/steps/DataWriterProcessorStepImpl.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/steps/DataWriterProcessorStepImpl.java b/processing/src/main/java/org/apache/carbondata/processing/loading/steps/DataWriterProcessorStepImpl.java index d321405..58009af 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/loading/steps/DataWriterProcessorStepImpl.java +++ b/processing/src/main/java/org/apache/carbondata/processing/loading/steps/DataWriterProcessorStepImpl.java @@ -26,6 +26,7 @@ import org.apache.carbondata.core.datastore.row.CarbonRow; import org.apache.carbondata.core.keygenerator.KeyGenException; import org.apache.carbondata.core.metadata.CarbonTableIdentifier; import org.apache.carbondata.core.util.CarbonTimeStatisticsFactory; +import org.apache.carbondata.core.util.path.CarbonTablePath; import org.apache.carbondata.processing.loading.AbstractDataLoadProcessorStep; import org.apache.carbondata.processing.loading.CarbonDataLoadConfiguration; import org.apache.carbondata.processing.loading.DataField; @@ -65,21 +66,21 @@ public class DataWriterProcessorStepImpl extends AbstractDataLoadProcessorStep { child.initialize(); } - private String[] getStoreLocation(CarbonTableIdentifier tableIdentifier, String partitionId) { + private String[] getStoreLocation(CarbonTableIdentifier tableIdentifier) { String[] storeLocation = CarbonDataProcessorUtil .getLocalDataFolderLocation(tableIdentifier.getDatabaseName(), - tableIdentifier.getTableName(), String.valueOf(configuration.getTaskNo()), partitionId, - configuration.getSegmentId() + "", false, false); + tableIdentifier.getTableName(), String.valueOf(configuration.getTaskNo()), + configuration.getSegmentId(), false, false); CarbonDataProcessorUtil.createLocations(storeLocation); return storeLocation; } - public CarbonFactDataHandlerModel getDataHandlerModel(int partitionId) { + public CarbonFactDataHandlerModel getDataHandlerModel() { CarbonTableIdentifier tableIdentifier = configuration.getTableIdentifier().getCarbonTableIdentifier(); - String[] storeLocation = getStoreLocation(tableIdentifier, String.valueOf(partitionId)); + String[] storeLocation = getStoreLocation(tableIdentifier); return CarbonFactDataHandlerModel.createCarbonFactDataHandlerModel(configuration, - storeLocation, partitionId, 0); + storeLocation, 0, 0); } @Override public Iterator<CarbonRowBatch>[] execute() throws CarbonDataLoadingException { @@ -89,11 +90,11 @@ public class DataWriterProcessorStepImpl extends AbstractDataLoadProcessorStep { String tableName = tableIdentifier.getTableName(); try { CarbonTimeStatisticsFactory.getLoadStatisticsInstance() - .recordDictionaryValue2MdkAdd2FileTime(configuration.getPartitionId(), + .recordDictionaryValue2MdkAdd2FileTime(CarbonTablePath.DEPRECATED_PATITION_ID, System.currentTimeMillis()); int i = 0; for (Iterator<CarbonRowBatch> iterator : iterators) { - String[] storeLocation = getStoreLocation(tableIdentifier, String.valueOf(i)); + String[] storeLocation = getStoreLocation(tableIdentifier); CarbonFactDataHandlerModel model = CarbonFactDataHandlerModel .createCarbonFactDataHandlerModel(configuration, storeLocation, i, 0); @@ -147,10 +148,11 @@ public class DataWriterProcessorStepImpl extends AbstractDataLoadProcessorStep { CarbonTimeStatisticsFactory.getLoadStatisticsInstance().recordTotalRecords(rowCounter.get()); processingComplete(dataHandler); CarbonTimeStatisticsFactory.getLoadStatisticsInstance() - .recordDictionaryValue2MdkAdd2FileTime(configuration.getPartitionId(), + .recordDictionaryValue2MdkAdd2FileTime(CarbonTablePath.DEPRECATED_PATITION_ID, System.currentTimeMillis()); CarbonTimeStatisticsFactory.getLoadStatisticsInstance() - .recordMdkGenerateTotalTime(configuration.getPartitionId(), System.currentTimeMillis()); + .recordMdkGenerateTotalTime(CarbonTablePath.DEPRECATED_PATITION_ID, + System.currentTimeMillis()); } private void processingComplete(CarbonFactHandler dataHandler) throws CarbonDataLoadingException { http://git-wip-us.apache.org/repos/asf/carbondata/blob/5663e916/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonDataMergerUtil.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonDataMergerUtil.java b/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonDataMergerUtil.java index 3f1430d..c141636 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonDataMergerUtil.java +++ b/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonDataMergerUtil.java @@ -321,7 +321,7 @@ public final class CarbonDataMergerUtil { // create entry for merged one. LoadMetadataDetails loadMetadataDetails = new LoadMetadataDetails(); - loadMetadataDetails.setPartitionCount(carbonLoadModel.getPartitionId()); + loadMetadataDetails.setPartitionCount(CarbonTablePath.DEPRECATED_PATITION_ID); loadMetadataDetails.setSegmentStatus(SegmentStatus.SUCCESS); long loadEnddate = CarbonUpdateUtil.readCurrentTime(); loadMetadataDetails.setLoadEndTime(loadEnddate); @@ -686,7 +686,7 @@ public final class CarbonDataMergerUtil { CarbonTableIdentifier carbonTableIdentifier, String segmentId) { CarbonTablePath carbonTablePath = CarbonStorePath.getCarbonTablePath(tablePath, carbonTableIdentifier); - return carbonTablePath.getCarbonDataDirectoryPath("0", segmentId); + return carbonTablePath.getCarbonDataDirectoryPath(segmentId); } @@ -1047,7 +1047,7 @@ public final class CarbonDataMergerUtil { CarbonTablePath carbonTablePath = CarbonStorePath.getCarbonTablePath(absoluteTableIdentifier); - String segmentPath = carbonTablePath.getCarbonDataDirectoryPath("0", seg.getSegmentNo()); + String segmentPath = carbonTablePath.getCarbonDataDirectoryPath(seg.getSegmentNo()); CarbonFile segDir = FileFactory.getCarbonFile(segmentPath, FileFactory.getFileType(segmentPath)); CarbonFile[] allSegmentFiles = segDir.listFiles(); http://git-wip-us.apache.org/repos/asf/carbondata/blob/5663e916/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java b/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java index e7c4502..a7b1711 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java +++ b/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java @@ -371,7 +371,7 @@ public class CompactionResultSortProcessor extends AbstractResultProcessor { return SortParameters .createSortParameters(carbonTable, carbonLoadModel.getDatabaseName(), tableName, dimensionColumnCount, segmentProperties.getComplexDimensions().size(), measureCount, - noDictionaryCount, carbonLoadModel.getPartitionId(), segmentId, + noDictionaryCount, segmentId, carbonLoadModel.getTaskNo(), noDictionaryColMapping, true); } @@ -408,7 +408,7 @@ public class CompactionResultSortProcessor extends AbstractResultProcessor { } else { carbonStoreLocation = CarbonDataProcessorUtil .createCarbonStoreLocation(carbonTable.getTablePath(), carbonLoadModel.getDatabaseName(), - tableName, carbonLoadModel.getPartitionId(), carbonLoadModel.getSegmentId()); + tableName, carbonLoadModel.getSegmentId()); } CarbonFactDataHandlerModel carbonFactDataHandlerModel = CarbonFactDataHandlerModel .getCarbonFactDataHandlerModel(carbonLoadModel, carbonTable, segmentProperties, tableName, @@ -430,7 +430,7 @@ public class CompactionResultSortProcessor extends AbstractResultProcessor { */ private void initTempStoreLocation() { tempStoreLocation = CarbonDataProcessorUtil - .getLocalDataFolderLocation(carbonTable, tableName, carbonLoadModel.getTaskNo(), - carbonLoadModel.getPartitionId(), segmentId, true, false); + .getLocalDataFolderLocation(carbonTable, carbonLoadModel.getTaskNo(), + segmentId, true, false); } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/5663e916/processing/src/main/java/org/apache/carbondata/processing/merger/RowResultMergerProcessor.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/apache/carbondata/processing/merger/RowResultMergerProcessor.java b/processing/src/main/java/org/apache/carbondata/processing/merger/RowResultMergerProcessor.java index b41829f..569e026 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/merger/RowResultMergerProcessor.java +++ b/processing/src/main/java/org/apache/carbondata/processing/merger/RowResultMergerProcessor.java @@ -78,7 +78,7 @@ public class RowResultMergerProcessor extends AbstractResultProcessor { } else { carbonStoreLocation = CarbonDataProcessorUtil .createCarbonStoreLocation(carbonTable.getTablePath(), loadModel.getDatabaseName(), - tableName, loadModel.getPartitionId(), loadModel.getSegmentId()); + tableName, loadModel.getSegmentId()); } CarbonFactDataHandlerModel carbonFactDataHandlerModel = CarbonFactDataHandlerModel .getCarbonFactDataHandlerModel(loadModel, carbonTable, segProp, tableName, http://git-wip-us.apache.org/repos/asf/carbondata/blob/5663e916/processing/src/main/java/org/apache/carbondata/processing/partition/spliter/RowResultProcessor.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/apache/carbondata/processing/partition/spliter/RowResultProcessor.java b/processing/src/main/java/org/apache/carbondata/processing/partition/spliter/RowResultProcessor.java index ff6ca93..92db4c5 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/partition/spliter/RowResultProcessor.java +++ b/processing/src/main/java/org/apache/carbondata/processing/partition/spliter/RowResultProcessor.java @@ -49,7 +49,7 @@ public class RowResultProcessor { String tableName = carbonTable.getTableName(); String carbonStoreLocation = CarbonDataProcessorUtil .createCarbonStoreLocation(carbonTable.getTablePath(), loadModel.getDatabaseName(), - tableName, loadModel.getPartitionId(), loadModel.getSegmentId()); + tableName, loadModel.getSegmentId()); CarbonFactDataHandlerModel carbonFactDataHandlerModel = CarbonFactDataHandlerModel.getCarbonFactDataHandlerModel(loadModel, carbonTable, segProp, tableName, tempStoreLocation, carbonStoreLocation); http://git-wip-us.apache.org/repos/asf/carbondata/blob/5663e916/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/SortParameters.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/SortParameters.java b/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/SortParameters.java index 98d150e..4d31f87 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/SortParameters.java +++ b/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/SortParameters.java @@ -26,6 +26,7 @@ import org.apache.carbondata.core.metadata.CarbonTableIdentifier; import org.apache.carbondata.core.metadata.datatype.DataType; import org.apache.carbondata.core.metadata.schema.table.CarbonTable; import org.apache.carbondata.core.util.CarbonProperties; +import org.apache.carbondata.core.util.path.CarbonTablePath; import org.apache.carbondata.processing.loading.CarbonDataLoadConfiguration; import org.apache.carbondata.processing.util.CarbonDataProcessorUtil; @@ -356,7 +357,7 @@ public class SortParameters implements Serializable { CarbonProperties carbonProperties = CarbonProperties.getInstance(); parameters.setDatabaseName(tableIdentifier.getDatabaseName()); parameters.setTableName(tableIdentifier.getTableName()); - parameters.setPartitionID(configuration.getPartitionId()); + parameters.setPartitionID("0"); parameters.setSegmentId(configuration.getSegmentId()); parameters.setTaskNo(configuration.getTaskNo()); parameters.setMeasureColCount(configuration.getMeasureCount()); @@ -392,10 +393,9 @@ public class SortParameters implements Serializable { LOGGER.info("File Buffer Size: " + parameters.getFileBufferSize()); - String[] carbonDataDirectoryPath = CarbonDataProcessorUtil - .getLocalDataFolderLocation(tableIdentifier.getDatabaseName(), - tableIdentifier.getTableName(), configuration.getTaskNo(), - configuration.getPartitionId(), configuration.getSegmentId(), false, false); + String[] carbonDataDirectoryPath = CarbonDataProcessorUtil.getLocalDataFolderLocation( + tableIdentifier.getDatabaseName(), tableIdentifier.getTableName(), + configuration.getTaskNo(), configuration.getSegmentId(), false, false); String[] sortTempDirs = CarbonDataProcessorUtil.arrayAppend(carbonDataDirectoryPath, File.separator, CarbonCommonConstants.SORT_TEMP_FILE_LOCATION); @@ -448,13 +448,13 @@ public class SortParameters implements Serializable { public static SortParameters createSortParameters(CarbonTable carbonTable, String databaseName, String tableName, int dimColCount, int complexDimColCount, int measureColCount, - int noDictionaryCount, String partitionID, String segmentId, String taskNo, + int noDictionaryCount, String segmentId, String taskNo, boolean[] noDictionaryColMaping, boolean isCompactionFlow) { SortParameters parameters = new SortParameters(); CarbonProperties carbonProperties = CarbonProperties.getInstance(); parameters.setDatabaseName(databaseName); parameters.setTableName(tableName); - parameters.setPartitionID(partitionID); + parameters.setPartitionID(CarbonTablePath.DEPRECATED_PATITION_ID); parameters.setSegmentId(segmentId); parameters.setTaskNo(taskNo); parameters.setMeasureColCount(measureColCount); @@ -486,7 +486,7 @@ public class SortParameters implements Serializable { LOGGER.info("File Buffer Size: " + parameters.getFileBufferSize()); String[] carbonDataDirectoryPath = CarbonDataProcessorUtil - .getLocalDataFolderLocation(databaseName, tableName, taskNo, partitionID, segmentId, + .getLocalDataFolderLocation(databaseName, tableName, taskNo, segmentId, isCompactionFlow, false); String[] sortTempDirs = CarbonDataProcessorUtil.arrayAppend(carbonDataDirectoryPath, File.separator, CarbonCommonConstants.SORT_TEMP_FILE_LOCATION); http://git-wip-us.apache.org/repos/asf/carbondata/blob/5663e916/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java index d77fcab..732a7e8 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java +++ b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java @@ -339,8 +339,7 @@ public class CarbonFactDataHandlerModel { AbsoluteTableIdentifier absoluteTableIdentifier = configuration.getTableIdentifier(); CarbonTablePath carbonTablePath = CarbonStorePath.getCarbonTablePath(absoluteTableIdentifier); String carbonDataDirectoryPath = carbonTablePath - .getCarbonDataDirectoryPath(configuration.getPartitionId(), - configuration.getSegmentId() + ""); + .getCarbonDataDirectoryPath(configuration.getSegmentId() + ""); CarbonUtil.checkAndCreateFolder(carbonDataDirectoryPath); return carbonDataDirectoryPath; } http://git-wip-us.apache.org/repos/asf/carbondata/blob/5663e916/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java index cfc9fa3..8a6cbe4 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java +++ b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java @@ -124,17 +124,15 @@ public final class CarbonDataProcessorUtil { * * @param carbonTable * @param taskId - * @param partitionId * @param segmentId * @param isCompactionFlow * @param isAltPartitionFlow * @return */ - public static String[] getLocalDataFolderLocation(CarbonTable carbonTable, String tableName, - String taskId, String partitionId, String segmentId, boolean isCompactionFlow, - boolean isAltPartitionFlow) { + public static String[] getLocalDataFolderLocation(CarbonTable carbonTable, + String taskId, String segmentId, boolean isCompactionFlow, boolean isAltPartitionFlow) { String tempLocationKey = - getTempStoreLocationKey(carbonTable.getDatabaseName(), tableName, + getTempStoreLocationKey(carbonTable.getDatabaseName(), carbonTable.getTableName(), segmentId, taskId, isCompactionFlow, isAltPartitionFlow); String baseTempStorePath = CarbonProperties.getInstance() .getProperty(tempLocationKey); @@ -152,8 +150,7 @@ public final class CarbonDataProcessorUtil { String tmpStore = baseTmpStorePathArray[i]; CarbonTablePath carbonTablePath = CarbonStorePath.getCarbonTablePath(tmpStore, carbonTable.getCarbonTableIdentifier()); - String carbonDataDirectoryPath = - carbonTablePath.getCarbonDataDirectoryPath(partitionId, segmentId + ""); + String carbonDataDirectoryPath = carbonTablePath.getCarbonDataDirectoryPath(segmentId); localDataFolderLocArray[i] = carbonDataDirectoryPath + File.separator + taskId; } @@ -166,17 +163,16 @@ public final class CarbonDataProcessorUtil { * @param databaseName * @param tableName * @param taskId - * @param partitionId * @param segmentId * @param isCompactionFlow * @param isAltPartitionFlow * @return */ public static String[] getLocalDataFolderLocation(String databaseName, String tableName, - String taskId, String partitionId, String segmentId, boolean isCompactionFlow, + String taskId, String segmentId, boolean isCompactionFlow, boolean isAltPartitionFlow) { CarbonTable carbonTable = CarbonMetadata.getInstance().getCarbonTable(databaseName, tableName); - return getLocalDataFolderLocation(carbonTable, tableName, taskId, partitionId, + return getLocalDataFolderLocation(carbonTable, taskId, segmentId, isCompactionFlow, isAltPartitionFlow); } @@ -401,13 +397,13 @@ public final class CarbonDataProcessorUtil { * @return data directory path */ public static String createCarbonStoreLocation(String factStoreLocation, - String databaseName, String tableName, String partitionId, String segmentId) { + String databaseName, String tableName, String segmentId) { CarbonTable carbonTable = CarbonMetadata.getInstance().getCarbonTable(databaseName, tableName); CarbonTableIdentifier carbonTableIdentifier = carbonTable.getCarbonTableIdentifier(); CarbonTablePath carbonTablePath = CarbonStorePath.getCarbonTablePath(factStoreLocation, carbonTableIdentifier); String carbonDataDirectoryPath = - carbonTablePath.getCarbonDataDirectoryPath(partitionId, segmentId); + carbonTablePath.getCarbonDataDirectoryPath(segmentId); return carbonDataDirectoryPath; } http://git-wip-us.apache.org/repos/asf/carbondata/blob/5663e916/processing/src/main/java/org/apache/carbondata/processing/util/CarbonLoaderUtil.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonLoaderUtil.java b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonLoaderUtil.java index 23f9aa8..d958937 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonLoaderUtil.java +++ b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonLoaderUtil.java @@ -80,10 +80,8 @@ public final class CarbonLoaderUtil { CarbonTablePath carbonTablePath = CarbonStorePath .getCarbonTablePath(loadModel.getTablePath(), carbonTable.getCarbonTableIdentifier()); - for (int i = 0; i < carbonTable.getPartitionCount(); i++) { - String segmentPath = carbonTablePath.getCarbonDataDirectoryPath(i + "", currentLoad + ""); - deleteStorePath(segmentPath); - } + String segmentPath = carbonTablePath.getCarbonDataDirectoryPath(currentLoad + ""); + deleteStorePath(segmentPath); } /** @@ -103,7 +101,7 @@ public final class CarbonLoaderUtil { int fileCount = 0; int partitionCount = carbonTable.getPartitionCount(); for (int i = 0; i < partitionCount; i++) { - String segmentPath = carbonTablePath.getCarbonDataDirectoryPath(i + "", + String segmentPath = carbonTablePath.getCarbonDataDirectoryPath( currentLoad + ""); CarbonFile carbonFile = FileFactory.getCarbonFile(segmentPath, FileFactory.getFileType(segmentPath)); @@ -386,7 +384,7 @@ public final class CarbonLoaderUtil { private static void addToStaleFolders(CarbonTablePath carbonTablePath, List<CarbonFile> staleFolders, LoadMetadataDetails entry) throws IOException { - String path = carbonTablePath.getCarbonDataDirectoryPath("0", entry.getLoadName()); + String path = carbonTablePath.getCarbonDataDirectoryPath(entry.getLoadName()); // add to the deletion list only if file exist else HDFS file system will throw // exception while deleting the file if file path does not exist if (FileFactory.isFileExist(path, FileFactory.getFileType(path))) { @@ -967,7 +965,7 @@ public final class CarbonLoaderUtil { CarbonTableIdentifier carbonTableIdentifier = carbonTable.getCarbonTableIdentifier(); CarbonTablePath carbonTablePath = CarbonStorePath.getCarbonTablePath(carbonTable.getTablePath(), carbonTableIdentifier); - String segmentFolder = carbonTablePath.getCarbonDataDirectoryPath("0", segmentId); + String segmentFolder = carbonTablePath.getCarbonDataDirectoryPath(segmentId); CarbonUtil.checkAndCreateFolder(segmentFolder); } http://git-wip-us.apache.org/repos/asf/carbondata/blob/5663e916/processing/src/main/java/org/apache/carbondata/processing/util/DeleteLoadFolders.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/apache/carbondata/processing/util/DeleteLoadFolders.java b/processing/src/main/java/org/apache/carbondata/processing/util/DeleteLoadFolders.java index 52b9f52..288cd54 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/util/DeleteLoadFolders.java +++ b/processing/src/main/java/org/apache/carbondata/processing/util/DeleteLoadFolders.java @@ -51,15 +51,14 @@ public final class DeleteLoadFolders { * returns segment path * * @param absoluteTableIdentifier - * @param partitionId * @param oneLoad * @return */ private static String getSegmentPath(AbsoluteTableIdentifier absoluteTableIdentifier, - int partitionId, LoadMetadataDetails oneLoad) { + LoadMetadataDetails oneLoad) { CarbonTablePath carbon = CarbonStorePath.getCarbonTablePath(absoluteTableIdentifier); String segmentId = oneLoad.getLoadName(); - return carbon.getCarbonDataDirectoryPath("" + partitionId, segmentId); + return carbon.getCarbonDataDirectoryPath(segmentId); } public static void physicalFactAndMeasureMetadataDeletion( @@ -74,7 +73,7 @@ public final class DeleteLoadFolders { .deleteSegment(absoluteTableIdentifier.getTablePath(), oneLoad.getSegmentFile(), specs); } else { - String path = getSegmentPath(absoluteTableIdentifier, 0, oneLoad); + String path = getSegmentPath(absoluteTableIdentifier, oneLoad); boolean status = false; if (FileFactory.isFileExist(path, FileFactory.getFileType(path))) { CarbonFile file = FileFactory.getCarbonFile(path, FileFactory.getFileType(path)); http://git-wip-us.apache.org/repos/asf/carbondata/blob/5663e916/processing/src/test/java/org/apache/carbondata/processing/StoreCreator.java ---------------------------------------------------------------------- diff --git a/processing/src/test/java/org/apache/carbondata/processing/StoreCreator.java b/processing/src/test/java/org/apache/carbondata/processing/StoreCreator.java index e93227d..12b00db 100644 --- a/processing/src/test/java/org/apache/carbondata/processing/StoreCreator.java +++ b/processing/src/test/java/org/apache/carbondata/processing/StoreCreator.java @@ -163,7 +163,6 @@ public class StoreCreator { loadModel.setCsvHeaderColumns(loadModel.getCsvHeader().split(",")); loadModel.setTaskNo("0"); loadModel.setSegmentId("0"); - loadModel.setPartitionId("0"); loadModel.setFactTimeStamp(System.currentTimeMillis()); loadModel.setMaxColumns("10"); http://git-wip-us.apache.org/repos/asf/carbondata/blob/5663e916/streaming/src/main/scala/org/apache/carbondata/streaming/StreamHandoffRDD.scala ---------------------------------------------------------------------- diff --git a/streaming/src/main/scala/org/apache/carbondata/streaming/StreamHandoffRDD.scala b/streaming/src/main/scala/org/apache/carbondata/streaming/StreamHandoffRDD.scala index a46ced5..5e0fce5 100644 --- a/streaming/src/main/scala/org/apache/carbondata/streaming/StreamHandoffRDD.scala +++ b/streaming/src/main/scala/org/apache/carbondata/streaming/StreamHandoffRDD.scala @@ -110,7 +110,6 @@ class StreamHandoffRDD[K, V]( split: Partition, context: TaskContext ): Iterator[(K, V)] = { - carbonLoadModel.setPartitionId("0") carbonLoadModel.setTaskNo("" + split.index) val carbonTable = carbonLoadModel.getCarbonDataLoadSchema.getCarbonTable CarbonMetadata.getInstance().addCarbonTable(carbonTable) http://git-wip-us.apache.org/repos/asf/carbondata/blob/5663e916/streaming/src/main/scala/org/apache/carbondata/streaming/StreamSinkFactory.scala ---------------------------------------------------------------------- diff --git a/streaming/src/main/scala/org/apache/carbondata/streaming/StreamSinkFactory.scala b/streaming/src/main/scala/org/apache/carbondata/streaming/StreamSinkFactory.scala index 3366f51..75fcfb0 100644 --- a/streaming/src/main/scala/org/apache/carbondata/streaming/StreamSinkFactory.scala +++ b/streaming/src/main/scala/org/apache/carbondata/streaming/StreamSinkFactory.scala @@ -136,7 +136,7 @@ object StreamSinkFactory { FileFactory.mkdirs(carbonTablePath.getMetadataDirectoryPath, fileType) } val segmentId = StreamSegment.open(carbonTable) - val segmentDir = carbonTablePath.getSegmentDir("0", segmentId) + val segmentDir = carbonTablePath.getSegmentDir(segmentId) if (FileFactory.isFileExist(segmentDir, fileType)) { // recover fault StreamSegment.recoverSegmentIfRequired(segmentDir) http://git-wip-us.apache.org/repos/asf/carbondata/blob/5663e916/streaming/src/main/scala/org/apache/spark/sql/execution/streaming/CarbonAppendableStreamSink.scala ---------------------------------------------------------------------- diff --git a/streaming/src/main/scala/org/apache/spark/sql/execution/streaming/CarbonAppendableStreamSink.scala b/streaming/src/main/scala/org/apache/spark/sql/execution/streaming/CarbonAppendableStreamSink.scala index 312d24e..4f1c999 100644 --- a/streaming/src/main/scala/org/apache/spark/sql/execution/streaming/CarbonAppendableStreamSink.scala +++ b/streaming/src/main/scala/org/apache/spark/sql/execution/streaming/CarbonAppendableStreamSink.scala @@ -166,12 +166,12 @@ class CarbonAppendableStreamSink( * if the directory size of current segment beyond the threshold, hand off new segment */ private def checkOrHandOffSegment(): Unit = { - val segmentDir = carbonTablePath.getSegmentDir("0", currentSegmentId) + val segmentDir = carbonTablePath.getSegmentDir(currentSegmentId) val fileType = FileFactory.getFileType(segmentDir) if (segmentMaxSize <= StreamSegment.size(segmentDir)) { val newSegmentId = StreamSegment.close(carbonTable, currentSegmentId) currentSegmentId = newSegmentId - val newSegmentDir = carbonTablePath.getSegmentDir("0", currentSegmentId) + val newSegmentDir = carbonTablePath.getSegmentDir(currentSegmentId) FileFactory.mkdirs(newSegmentDir, fileType) // TODO trigger hand off operation @@ -271,14 +271,14 @@ object CarbonAppendableStreamSink { // update data file info in index file val tablePath = CarbonStorePath.getCarbonTablePath(carbonTable.getAbsoluteTableIdentifier) - StreamSegment.updateIndexFile(tablePath.getSegmentDir("0", segmentId)) + StreamSegment.updateIndexFile(tablePath.getSegmentDir(segmentId)) } catch { // catch fault of executor side case t: Throwable => val tablePath = CarbonStorePath.getCarbonTablePath(carbonTable.getAbsoluteTableIdentifier) - val segmentDir = tablePath.getSegmentDir("0", segmentId) + val segmentDir = tablePath.getSegmentDir(segmentId) StreamSegment.recoverSegmentIfRequired(segmentDir) LOGGER.error(t, s"Aborting job ${ job.getJobID }.") committer.abortJob(job)
