Repository: carbondata Updated Branches: refs/heads/master 2f85381f8 -> 61788353d
http://git-wip-us.apache.org/repos/asf/carbondata/blob/61788353/hadoop/src/main/java/org/apache/carbondata/hadoop/util/BlockLevelTraverser.java ---------------------------------------------------------------------- diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/util/BlockLevelTraverser.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/util/BlockLevelTraverser.java deleted file mode 100644 index 2f28861..0000000 --- a/hadoop/src/main/java/org/apache/carbondata/hadoop/util/BlockLevelTraverser.java +++ /dev/null @@ -1,87 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.carbondata.hadoop.util; - -import java.util.Map; - -import org.apache.carbondata.core.datastore.DataRefNode; -import org.apache.carbondata.core.datastore.DataRefNodeFinder; -import org.apache.carbondata.core.datastore.IndexKey; -import org.apache.carbondata.core.datastore.block.AbstractIndex; -import org.apache.carbondata.core.datastore.impl.btree.BTreeDataRefNodeFinder; -import org.apache.carbondata.core.datastore.impl.btree.BlockBTreeLeafNode; -import org.apache.carbondata.core.keygenerator.KeyGenException; -import org.apache.carbondata.core.mutate.CarbonUpdateUtil; -import org.apache.carbondata.core.mutate.SegmentUpdateDetails; -import org.apache.carbondata.core.scan.filter.FilterUtil; -import org.apache.carbondata.core.statusmanager.SegmentUpdateStatusManager; -import org.apache.carbondata.core.util.path.CarbonTablePath; - -/** - * Block level traverser - */ -public class BlockLevelTraverser { - - /** - * - * @param abstractIndex - * @param blockRowMap - * @param segId - * @param updateStatusManager - * @throws KeyGenException - */ - public long getBlockRowMapping(AbstractIndex abstractIndex, Map<String, Long> blockRowMap, - String segId, SegmentUpdateStatusManager updateStatusManager) - throws KeyGenException { - - IndexKey searchStartKey = - FilterUtil.prepareDefaultStartIndexKey(abstractIndex.getSegmentProperties()); - - DataRefNodeFinder blockFinder = new BTreeDataRefNodeFinder( - abstractIndex.getSegmentProperties().getEachDimColumnValueSize(), - abstractIndex.getSegmentProperties().getNumberOfSortColumns(), - abstractIndex.getSegmentProperties().getNumberOfNoDictSortColumns()); - DataRefNode currentBlock = - blockFinder.findFirstDataBlock(abstractIndex.getDataRefNode(), searchStartKey); - - long count = 0; - - while (currentBlock != null) { - - String blockName = ((BlockBTreeLeafNode) currentBlock).getTableBlockInfo().getFilePath(); - blockName = CarbonTablePath.getCarbonDataFileName(blockName); - blockName = blockName + CarbonTablePath.getCarbonDataExtension(); - - long rowCount = currentBlock.numRows(); - - String key = CarbonUpdateUtil.getSegmentBlockNameKey(segId, blockName); - - // if block is invalid then dont add the count - SegmentUpdateDetails details = updateStatusManager.getDetailsForABlock(key); - - if (null == details || !CarbonUpdateUtil.isBlockInvalid(details.getSegmentStatus())) { - blockRowMap.put(key, rowCount); - count++; - } - currentBlock = currentBlock.getNextDataRefNode(); - } - - return count; - } - -} http://git-wip-us.apache.org/repos/asf/carbondata/blob/61788353/processing/src/test/java/org/apache/carbondata/carbon/datastore/BlockIndexStoreTest.java ---------------------------------------------------------------------- diff --git a/processing/src/test/java/org/apache/carbondata/carbon/datastore/BlockIndexStoreTest.java b/processing/src/test/java/org/apache/carbondata/carbon/datastore/BlockIndexStoreTest.java deleted file mode 100644 index d30891a..0000000 --- a/processing/src/test/java/org/apache/carbondata/carbon/datastore/BlockIndexStoreTest.java +++ /dev/null @@ -1,108 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.carbondata.carbon.datastore; - -import java.io.File; -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.Callable; - -import org.apache.carbondata.common.logging.LogService; -import org.apache.carbondata.common.logging.LogServiceFactory; -import org.apache.carbondata.core.cache.CacheProvider; -import org.apache.carbondata.core.cache.CacheType; -import org.apache.carbondata.core.constants.CarbonCommonConstants; -import org.apache.carbondata.core.datastore.BlockIndexStore; -import org.apache.carbondata.core.datastore.block.AbstractIndex; -import org.apache.carbondata.core.datastore.block.TableBlockInfo; -import org.apache.carbondata.core.datastore.block.TableBlockUniqueIdentifier; -import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier; -import org.apache.carbondata.core.util.CarbonProperties; -import org.apache.carbondata.processing.StoreCreator; - -import junit.framework.TestCase; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.Test; - -public class BlockIndexStoreTest extends TestCase { - - // private BlockIndexStore indexStore; - BlockIndexStore<TableBlockUniqueIdentifier, AbstractIndex> cache; - - private static final LogService LOGGER = - LogServiceFactory.getLogService(BlockIndexStoreTest.class.getName()); - - @BeforeClass public void setUp() { - StoreCreator.createCarbonStore(); - CarbonProperties.getInstance(). - addProperty(CarbonCommonConstants.CARBON_MAX_DRIVER_LRU_CACHE_SIZE, "10"); - CacheProvider cacheProvider = CacheProvider.getInstance(); - cache = (BlockIndexStore) cacheProvider.createCache(CacheType.EXECUTOR_BTREE); - } - - @AfterClass public void tearDown() { - } - - @Test public void testEmpty() { - - } - - private List<TableBlockUniqueIdentifier> getTableBlockUniqueIdentifierList(List<TableBlockInfo> tableBlockInfos, - AbsoluteTableIdentifier absoluteTableIdentifier) { - List<TableBlockUniqueIdentifier> tableBlockUniqueIdentifiers = new ArrayList<>(); - for (TableBlockInfo tableBlockInfo : tableBlockInfos) { - tableBlockUniqueIdentifiers.add(new TableBlockUniqueIdentifier(absoluteTableIdentifier, tableBlockInfo)); - } - return tableBlockUniqueIdentifiers; - } - - private class BlockLoaderThread implements Callable<Void> { - private List<TableBlockInfo> tableBlockInfoList; - private AbsoluteTableIdentifier absoluteTableIdentifier; - - public BlockLoaderThread(List<TableBlockInfo> tableBlockInfoList, - AbsoluteTableIdentifier absoluteTableIdentifier) { - this.tableBlockInfoList = tableBlockInfoList; - this.absoluteTableIdentifier = absoluteTableIdentifier; - } - - @Override public Void call() throws Exception { - List<TableBlockUniqueIdentifier> tableBlockUniqueIdentifierList = - getTableBlockUniqueIdentifierList(tableBlockInfoList, absoluteTableIdentifier); - cache.getAll(tableBlockUniqueIdentifierList); - return null; - } - - } - - private static File getPartFile() { - String path = StoreCreator.getIdentifier().getTablePath() - + "/Fact/Part0/Segment_0"; - File file = new File(path); - File[] files = file.listFiles(); - File part = null; - for (int i = 0; i < files.length; i++) { - if (files[i].getName().startsWith("part")) { - part = files[i]; - } - } - return part; - } - -}
