Repository: carbondata Updated Branches: refs/heads/master 0112ed096 -> d8562e5bd
[CARBONDATA-2706][BloomDataMap] clear bloom index files after segment is deleted clear bloom index files after corresponding segment is deleted and cleaned This closes #2461 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/d8562e5b Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/d8562e5b Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/d8562e5b Branch: refs/heads/master Commit: d8562e5bd2bb7ed2ecb6a56b905e6b219b539caf Parents: 0112ed0 Author: xuchuanyin <[email protected]> Authored: Mon Jul 9 11:36:13 2018 +0800 Committer: Jacky Li <[email protected]> Committed: Tue Jul 10 19:26:52 2018 +0800 ---------------------------------------------------------------------- .../carbondata/core/datamap/TableDataMap.java | 8 +++++ .../core/datamap/dev/DataMapFactory.java | 5 ++++ .../blockletindex/BlockletDataMapFactory.java | 5 ++++ .../carbondata/core/util/DeleteLoadFolders.java | 28 +++++++++++++++++- .../bloom/BloomCoarseGrainDataMapFactory.java | 27 +++++++++++------ .../examples/MinMaxIndexDataMapFactory.java | 5 ++++ .../lucene/LuceneDataMapFactoryBase.java | 30 +++++++++++-------- .../testsuite/datamap/CGDataMapTestCase.scala | 7 +++++ .../testsuite/datamap/DataMapWriterSuite.scala | 7 +++++ .../testsuite/datamap/FGDataMapTestCase.scala | 7 +++++ .../testsuite/datamap/TestDataMapStatus.scala | 7 +++++ .../TestInsertAndOtherCommandConcurrent.scala | 7 +++++ .../management/CarbonCleanFilesCommand.scala | 5 ++++ .../BloomCoarseGrainDataMapFunctionSuite.scala | 31 ++++++++++++++++++++ 14 files changed, 157 insertions(+), 22 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/carbondata/blob/d8562e5b/core/src/main/java/org/apache/carbondata/core/datamap/TableDataMap.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/TableDataMap.java b/core/src/main/java/org/apache/carbondata/core/datamap/TableDataMap.java index cb7ec03..89a4c86 100644 --- a/core/src/main/java/org/apache/carbondata/core/datamap/TableDataMap.java +++ b/core/src/main/java/org/apache/carbondata/core/datamap/TableDataMap.java @@ -208,6 +208,14 @@ public final class TableDataMap extends OperationEventListener { } /** + * delete only the datamaps of the segments + */ + public void deleteDatamapData(List<Segment> segments) { + for (Segment segment: segments) { + dataMapFactory.deleteDatamapData(segment); + } + } + /** * delete datamap data if any */ public void deleteDatamapData() { http://git-wip-us.apache.org/repos/asf/carbondata/blob/d8562e5b/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapFactory.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapFactory.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapFactory.java index ff4ef72..b115462 100644 --- a/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapFactory.java +++ b/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapFactory.java @@ -111,6 +111,11 @@ public abstract class DataMapFactory<T extends DataMap> { public abstract DataMapLevel getDataMapLevel(); /** + * delete datamap data in the specified segment + */ + public abstract void deleteDatamapData(Segment segment); + + /** * delete datamap data if any */ public abstract void deleteDatamapData(); http://git-wip-us.apache.org/repos/asf/carbondata/blob/d8562e5b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapFactory.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapFactory.java b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapFactory.java index 51e6e21..175a2a4 100644 --- a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapFactory.java +++ b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapFactory.java @@ -330,6 +330,11 @@ public class BlockletDataMapFactory extends CoarseGrainDataMapFactory return null; } + @Override + public void deleteDatamapData(Segment segment) { + + } + @Override public void deleteDatamapData() { } http://git-wip-us.apache.org/repos/asf/carbondata/blob/d8562e5b/core/src/main/java/org/apache/carbondata/core/util/DeleteLoadFolders.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/util/DeleteLoadFolders.java b/core/src/main/java/org/apache/carbondata/core/util/DeleteLoadFolders.java index 25ba553..0433ba4 100644 --- a/core/src/main/java/org/apache/carbondata/core/util/DeleteLoadFolders.java +++ b/core/src/main/java/org/apache/carbondata/core/util/DeleteLoadFolders.java @@ -18,10 +18,14 @@ package org.apache.carbondata.core.util; import java.io.IOException; +import java.util.ArrayList; import java.util.List; import org.apache.carbondata.common.logging.LogService; import org.apache.carbondata.common.logging.LogServiceFactory; +import org.apache.carbondata.core.datamap.DataMapStoreManager; +import org.apache.carbondata.core.datamap.Segment; +import org.apache.carbondata.core.datamap.TableDataMap; import org.apache.carbondata.core.datastore.filesystem.CarbonFile; import org.apache.carbondata.core.datastore.filesystem.CarbonFileFilter; import org.apache.carbondata.core.datastore.impl.FileFactory; @@ -31,6 +35,7 @@ import org.apache.carbondata.core.locks.ICarbonLock; import org.apache.carbondata.core.locks.LockUsage; import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier; import org.apache.carbondata.core.metadata.SegmentFileStore; +import org.apache.carbondata.core.metadata.schema.table.CarbonTable; import org.apache.carbondata.core.mutate.CarbonUpdateUtil; import org.apache.carbondata.core.statusmanager.LoadMetadataDetails; import org.apache.carbondata.core.statusmanager.SegmentStatus; @@ -85,7 +90,22 @@ public final class DeleteLoadFolders { LoadMetadataDetails[] loadDetails, boolean isForceDelete, List<PartitionSpec> specs) { - for (LoadMetadataDetails oneLoad : loadDetails) { + CarbonTable carbonTable = DataMapStoreManager.getInstance().getCarbonTable( + absoluteTableIdentifier); + List<TableDataMap> indexDataMaps = new ArrayList<>(); + try { + for (TableDataMap dataMap : DataMapStoreManager.getInstance().getAllDataMap(carbonTable)) { + if (dataMap.getDataMapSchema().isIndexDataMap()) { + indexDataMaps.add(dataMap); + } + } + } catch (IOException e) { + LOGGER.warn(String.format( + "Failed to get datamaps for %s.%s, therefore the datamap files could not be cleaned.", + absoluteTableIdentifier.getDatabaseName(), absoluteTableIdentifier.getTableName())); + } + + for (final LoadMetadataDetails oneLoad : loadDetails) { if (checkIfLoadCanBeDeletedPhysically(oneLoad, isForceDelete)) { try { if (oneLoad.getSegmentFile() != null) { @@ -135,6 +155,12 @@ public final class DeleteLoadFolders { } } + List<Segment> segments = new ArrayList<>(1); + for (TableDataMap dataMap : indexDataMaps) { + segments.clear(); + segments.add(new Segment(oneLoad.getLoadName())); + dataMap.deleteDatamapData(segments); + } } catch (IOException e) { LOGGER.warn("Unable to delete the file as per delete command " + oneLoad.getLoadName()); } http://git-wip-us.apache.org/repos/asf/carbondata/blob/d8562e5b/datamap/bloom/src/main/java/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapFactory.java ---------------------------------------------------------------------- diff --git a/datamap/bloom/src/main/java/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapFactory.java b/datamap/bloom/src/main/java/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapFactory.java index 57c645f..aa6c5e1 100644 --- a/datamap/bloom/src/main/java/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapFactory.java +++ b/datamap/bloom/src/main/java/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapFactory.java @@ -345,22 +345,31 @@ public class BloomCoarseGrainDataMapFactory extends DataMapFactory<CoarseGrainDa } @Override + public void deleteDatamapData(Segment segment) { + try { + String segmentId = segment.getSegmentNo(); + String datamapPath = CarbonTablePath + .getDataMapStorePath(getCarbonTable().getTablePath(), segmentId, dataMapName); + if (FileFactory.isFileExist(datamapPath)) { + CarbonFile file = FileFactory.getCarbonFile(datamapPath, + FileFactory.getFileType(datamapPath)); + CarbonUtil.deleteFoldersAndFilesSilent(file); + } + } catch (IOException | InterruptedException ex) { + LOGGER.error("Failed to delete datamap for segment_" + segment.getSegmentNo()); + } + } + + @Override public void deleteDatamapData() { SegmentStatusManager ssm = new SegmentStatusManager(getCarbonTable().getAbsoluteTableIdentifier()); try { List<Segment> validSegments = ssm.getValidAndInvalidSegments().getValidSegments(); for (Segment segment : validSegments) { - String segmentId = segment.getSegmentNo(); - String datamapPath = CarbonTablePath - .getDataMapStorePath(getCarbonTable().getTablePath(), segmentId, dataMapName); - if (FileFactory.isFileExist(datamapPath)) { - CarbonFile file = FileFactory.getCarbonFile(datamapPath, - FileFactory.getFileType(datamapPath)); - CarbonUtil.deleteFoldersAndFilesSilent(file); - } + deleteDatamapData(segment); } - } catch (IOException | InterruptedException ex) { + } catch (IOException e) { LOGGER.error("drop datamap failed, failed to delete datamap directory"); } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/d8562e5b/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxIndexDataMapFactory.java ---------------------------------------------------------------------- diff --git a/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxIndexDataMapFactory.java b/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxIndexDataMapFactory.java index bc65a93..506b91a 100644 --- a/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxIndexDataMapFactory.java +++ b/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxIndexDataMapFactory.java @@ -156,6 +156,11 @@ public class MinMaxIndexDataMapFactory extends CoarseGrainDataMapFactory { return this.dataMapMeta; } + @Override + public void deleteDatamapData(Segment segment) { + + } + @Override public void deleteDatamapData() { } http://git-wip-us.apache.org/repos/asf/carbondata/blob/d8562e5b/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneDataMapFactoryBase.java ---------------------------------------------------------------------- diff --git a/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneDataMapFactoryBase.java b/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneDataMapFactoryBase.java index 5393746..8186841 100644 --- a/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneDataMapFactoryBase.java +++ b/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneDataMapFactoryBase.java @@ -181,19 +181,9 @@ abstract class LuceneDataMapFactoryBase<T extends DataMap> extends DataMapFactor try { List<Segment> validSegments = ssm.getValidAndInvalidSegments().getValidSegments(); for (Segment segment : validSegments) { - String segmentId = segment.getSegmentNo(); - String datamapPath = CarbonTablePath - .getDataMapStorePath(tableIdentifier.getTablePath(), segmentId, dataMapName); - if (FileFactory.isFileExist(datamapPath)) { - CarbonFile file = - FileFactory.getCarbonFile(datamapPath, FileFactory.getFileType(datamapPath)); - CarbonUtil.deleteFoldersAndFilesSilent(file); - } + deleteDatamapData(segment); } - } catch (IOException ex) { - throw new MalformedDataMapCommandException( - "drop datamap failed, failed to delete datamap directory"); - } catch (InterruptedException ex) { + } catch (IOException | RuntimeException ex) { throw new MalformedDataMapCommandException( "drop datamap failed, failed to delete datamap directory"); } @@ -270,6 +260,22 @@ abstract class LuceneDataMapFactoryBase<T extends DataMap> extends DataMapFactor } + @Override public void deleteDatamapData(Segment segment) { + try { + String segmentId = segment.getSegmentNo(); + String datamapPath = CarbonTablePath + .getDataMapStorePath(tableIdentifier.getTablePath(), segmentId, dataMapName); + if (FileFactory.isFileExist(datamapPath)) { + CarbonFile file = FileFactory.getCarbonFile(datamapPath, + FileFactory.getFileType(datamapPath)); + CarbonUtil.deleteFoldersAndFilesSilent(file); + } + } catch (IOException | InterruptedException ex) { + throw new RuntimeException( + "drop datamap failed, failed to delete datamap directory"); + } + } + @Override public void deleteDatamapData() { try { deleteDatamap(); http://git-wip-us.apache.org/repos/asf/carbondata/blob/d8562e5b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/CGDataMapTestCase.scala ---------------------------------------------------------------------- diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/CGDataMapTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/CGDataMapTestCase.scala index bc8b28b..9a7cb99 100644 --- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/CGDataMapTestCase.scala +++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/CGDataMapTestCase.scala @@ -136,6 +136,13 @@ class CGDataMapFactory( } /** + * delete datamap of the segment + */ + override def deleteDatamapData(segment: Segment): Unit = { + ??? + } + + /** * delete datamap data if any */ override def deleteDatamapData(): Unit = { http://git-wip-us.apache.org/repos/asf/carbondata/blob/d8562e5b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/DataMapWriterSuite.scala ---------------------------------------------------------------------- diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/DataMapWriterSuite.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/DataMapWriterSuite.scala index d4d78c9..434c363 100644 --- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/DataMapWriterSuite.scala +++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/DataMapWriterSuite.scala @@ -71,6 +71,13 @@ class C2DataMapFactory( } /** + * delete datamap of the segment + */ + override def deleteDatamapData(segment: Segment): Unit = { + ??? + } + + /** * delete datamap data if any */ override def deleteDatamapData(): Unit = { http://git-wip-us.apache.org/repos/asf/carbondata/blob/d8562e5b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/FGDataMapTestCase.scala ---------------------------------------------------------------------- diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/FGDataMapTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/FGDataMapTestCase.scala index 02ada2e..f4c12f7 100644 --- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/FGDataMapTestCase.scala +++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/FGDataMapTestCase.scala @@ -129,6 +129,13 @@ class FGDataMapFactory(carbonTable: CarbonTable, } /** + * delete datamap of the segment + */ + override def deleteDatamapData(segment: Segment): Unit = { + ??? + } + + /** * delete datamap data if any */ override def deleteDatamapData(): Unit = { http://git-wip-us.apache.org/repos/asf/carbondata/blob/d8562e5b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestDataMapStatus.scala ---------------------------------------------------------------------- diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestDataMapStatus.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestDataMapStatus.scala index c615078..0638a07 100644 --- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestDataMapStatus.scala +++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestDataMapStatus.scala @@ -267,6 +267,13 @@ class TestDataMapFactory( } /** + * delete datamap of the segment + */ + override def deleteDatamapData(segment: Segment): Unit = { + ??? + } + + /** * delete datamap data if any */ override def deleteDatamapData(): Unit = { http://git-wip-us.apache.org/repos/asf/carbondata/blob/d8562e5b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/iud/TestInsertAndOtherCommandConcurrent.scala ---------------------------------------------------------------------- diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/iud/TestInsertAndOtherCommandConcurrent.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/iud/TestInsertAndOtherCommandConcurrent.scala index 2441b53..dbbcd4d 100644 --- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/iud/TestInsertAndOtherCommandConcurrent.scala +++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/iud/TestInsertAndOtherCommandConcurrent.scala @@ -340,6 +340,13 @@ class WaitingDataMapFactory( } /** + * delete datamap of the segment + */ + override def deleteDatamapData(segment: Segment): Unit = { + + } + + /** * delete datamap data if any */ override def deleteDatamapData(): Unit = { http://git-wip-us.apache.org/repos/asf/carbondata/blob/d8562e5b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonCleanFilesCommand.scala ---------------------------------------------------------------------- diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonCleanFilesCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonCleanFilesCommand.scala index 1c4d068..9d0f149 100644 --- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonCleanFilesCommand.scala +++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonCleanFilesCommand.scala @@ -29,6 +29,7 @@ import org.apache.carbondata.api.CarbonStore import org.apache.carbondata.common.exceptions.sql.MalformedCarbonCommandException import org.apache.carbondata.common.logging.LogServiceFactory import org.apache.carbondata.core.constants.CarbonCommonConstants +import org.apache.carbondata.core.datamap.DataMapStoreManager import org.apache.carbondata.core.exception.ConcurrentOperationException import org.apache.carbondata.core.indexstore.PartitionSpec import org.apache.carbondata.core.metadata.schema.table.CarbonTable @@ -57,6 +58,10 @@ case class CarbonCleanFilesCommand( override def processMetadata(sparkSession: SparkSession): Seq[Row] = { carbonTable = CarbonEnv.getCarbonTable(databaseNameOp, tableName.get)(sparkSession) + val dms = carbonTable.getTableInfo.getDataMapSchemaList.asScala.map(_.getDataMapName) + val indexDms = DataMapStoreManager.getInstance.getAllDataMap(carbonTable).asScala + .filter(_.getDataMapSchema.isIndexDataMap) + if (carbonTable.hasAggregationDataMap) { cleanFileCommands = carbonTable.getTableInfo.getDataMapSchemaList.asScala.map { dataMapSchema => http://git-wip-us.apache.org/repos/asf/carbondata/blob/d8562e5b/integration/spark2/src/test/scala/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapFunctionSuite.scala ---------------------------------------------------------------------- diff --git a/integration/spark2/src/test/scala/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapFunctionSuite.scala b/integration/spark2/src/test/scala/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapFunctionSuite.scala index d0e2cbd..92943b2 100644 --- a/integration/spark2/src/test/scala/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapFunctionSuite.scala +++ b/integration/spark2/src/test/scala/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapFunctionSuite.scala @@ -2,12 +2,16 @@ package org.apache.carbondata.datamap.bloom import java.io.File +import org.apache.commons.io.FileUtils +import org.apache.spark.sql.CarbonEnv +import org.apache.spark.sql.test.Spark2TestQueryExecutor import org.apache.spark.sql.test.util.QueryTest import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach} import org.apache.carbondata.core.constants.CarbonCommonConstants import org.apache.carbondata.core.datamap.status.DataMapStatusManager import org.apache.carbondata.core.util.CarbonProperties +import org.apache.carbondata.core.util.path.CarbonTablePath import org.apache.carbondata.datamap.bloom.BloomCoarseGrainDataMapTestUtil.deleteFile import org.apache.carbondata.datamap.bloom.BloomCoarseGrainDataMapTestUtil.createFile import org.apache.carbondata.datamap.bloom.BloomCoarseGrainDataMapTestUtil.checkBasicQuery @@ -759,6 +763,33 @@ class BloomCoarseGrainDataMapFunctionSuite extends QueryTest with BeforeAndAfte sql(s"SELECT * FROM $normalTable WHERE isNull(c2)")) } + test("test bloom datamap: deleting & clearning segment will clear datamap files") { + sql(s"CREATE TABLE $bloomDMSampleTable(c1 string, c2 int, c3 string) STORED BY 'carbondata'") + sql( + s""" + | CREATE DATAMAP $dataMapName on table $bloomDMSampleTable + | using 'bloomfilter' + | DMPROPERTIES('index_columns'='c1, c2') + """.stripMargin) + sql(s"INSERT INTO $bloomDMSampleTable SELECT 'c1v1', 1, 'c3v1'") + sql(s"INSERT INTO $bloomDMSampleTable SELECT 'c1v2', 2, 'c3v2'") + + // two segments both has datamap files + val carbonTable = CarbonEnv.getCarbonTable(Option("default"), bloomDMSampleTable)(Spark2TestQueryExecutor.spark) + import scala.collection.JavaConverters._ + (0 to 1).foreach { segId => + val datamapPath = CarbonTablePath.getDataMapStorePath(carbonTable.getTablePath, segId.toString, dataMapName) + assert(FileUtils.listFiles(FileUtils.getFile(datamapPath), Array("bloomindex"), true).asScala.nonEmpty) + } + // delete and clean the first segment, the corresponding datamap files should be cleaned too + sql(s"DELETE FROM TABLE $bloomDMSampleTable WHERE SEGMENT.ID IN (0)") + sql(s"CLEAN FILES FOR TABLE $bloomDMSampleTable") + var datamapPath = CarbonTablePath.getDataMapStorePath(carbonTable.getTablePath, "0", dataMapName) + assert(!FileUtils.getFile(datamapPath).exists(), "index file of this segment has been deleted, should not exist") + datamapPath = CarbonTablePath.getDataMapStorePath(carbonTable.getTablePath, "1", dataMapName) + assert(FileUtils.listFiles(FileUtils.getFile(datamapPath), Array("bloomindex"), true).asScala.nonEmpty) + } + override def afterAll(): Unit = { deleteFile(bigFile) sql(s"DROP TABLE IF EXISTS $normalTable")
