Repository: carbondata Updated Branches: refs/heads/master 641ec098f -> 57b457153
[CARBONDATA-2716][DataMap] Add validation for datamap writer listener during data loading In some scenarios, while doing data loading, the loading will use the datamap writer listener that does not belong to this table. Here we will validate the tableName to make sure that the loading will only use the corresponding listerners. This closes #2471 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/57b45715 Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/57b45715 Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/57b45715 Branch: refs/heads/master Commit: 57b457153090de145d3b323a877139fbd65baa5a Parents: 641ec09 Author: xuchuanyin <xuchuan...@hust.edu.cn> Authored: Tue Jul 10 16:22:24 2018 +0800 Committer: Jacky Li <jacky.li...@qq.com> Committed: Wed Jul 11 16:00:29 2018 +0800 ---------------------------------------------------------------------- .../carbondata/processing/datamap/DataMapWriterListener.java | 8 ++++++++ .../store/writer/v3/CarbonFactDataWriterImplV3.java | 8 ++++++-- 2 files changed, 14 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/carbondata/blob/57b45715/processing/src/main/java/org/apache/carbondata/processing/datamap/DataMapWriterListener.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/apache/carbondata/processing/datamap/DataMapWriterListener.java b/processing/src/main/java/org/apache/carbondata/processing/datamap/DataMapWriterListener.java index 24fcf61..4207253 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/datamap/DataMapWriterListener.java +++ b/processing/src/main/java/org/apache/carbondata/processing/datamap/DataMapWriterListener.java @@ -34,6 +34,7 @@ import org.apache.carbondata.core.datamap.dev.DataMapFactory; import org.apache.carbondata.core.datamap.dev.DataMapWriter; import org.apache.carbondata.core.datastore.block.SegmentProperties; import org.apache.carbondata.core.datastore.page.ColumnPage; +import org.apache.carbondata.core.metadata.CarbonTableIdentifier; import org.apache.carbondata.core.metadata.schema.table.CarbonTable; import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn; import org.apache.carbondata.processing.store.TablePage; @@ -48,6 +49,12 @@ public class DataMapWriterListener { // list indexed column -> list of data map writer private Map<List<CarbonColumn>, List<DataMapWriter>> registry = new ConcurrentHashMap<>(); + // table for this listener + private CarbonTableIdentifier tblIdentifier; + + public CarbonTableIdentifier getTblIdentifier() { + return tblIdentifier; + } /** * register all datamap writer for specified table and segment @@ -62,6 +69,7 @@ public class DataMapWriterListener { throw new RuntimeException(e); } if (tableIndices != null) { + tblIdentifier = carbonTable.getCarbonTableIdentifier(); for (TableDataMap tableDataMap : tableIndices) { // register it only if it is not lazy datamap, for lazy datamap, user // will rebuild the datamap manually http://git-wip-us.apache.org/repos/asf/carbondata/blob/57b45715/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/CarbonFactDataWriterImplV3.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/CarbonFactDataWriterImplV3.java b/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/CarbonFactDataWriterImplV3.java index e562f26..dc6e443 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/CarbonFactDataWriterImplV3.java +++ b/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/CarbonFactDataWriterImplV3.java @@ -151,7 +151,9 @@ public class CarbonFactDataWriterImplV3 extends AbstractFactDataWriter { private void addPageData(TablePage tablePage) throws IOException { blockletDataHolder.addPage(tablePage); - if (listener != null) { + if (listener != null && + model.getDatabaseName().equalsIgnoreCase(listener.getTblIdentifier().getDatabaseName()) && + model.getTableName().equalsIgnoreCase(listener.getTblIdentifier().getTableName())) { if (pageId == 0) { listener.onBlockletStart(blockletId); } @@ -185,7 +187,9 @@ public class CarbonFactDataWriterImplV3 extends AbstractFactDataWriter { writeHeaderToFile(); } writeBlockletToFile(dataChunkBytes); - if (listener != null) { + if (listener != null && + model.getDatabaseName().equalsIgnoreCase(listener.getTblIdentifier().getDatabaseName()) && + model.getTableName().equalsIgnoreCase(listener.getTblIdentifier().getTableName())) { listener.onBlockletEnd(blockletId++); } pageId = 0;
