Repository: incubator-carbondata Updated Branches: refs/heads/presto f39ea5bbe -> 3bdda74fe
[CARBONDATA-872] Fix comment issues of integration/presto for easier reading fix comments Add comments for some classes and functions update annotation format delete checkstyle directory Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/10bddd88 Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/10bddd88 Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/10bddd88 Branch: refs/heads/presto Commit: 10bddd88fc0c2af0088bc02de3665ff7a308e87e Parents: f39ea5b Author: ffpeng90 <[email protected]> Authored: Wed Apr 5 23:00:50 2017 +0800 Committer: chenliang613 <[email protected]> Committed: Thu Apr 6 17:50:47 2017 +0530 ---------------------------------------------------------------------- .../presto/CarbondataColumnConstraint.java | 3 ++ .../presto/CarbondataColumnHandle.java | 6 --- .../presto/CarbondataConnectorFactory.java | 4 ++ .../carbondata/presto/CarbondataMetadata.java | 7 ---- .../presto/CarbondataRecordCursor.java | 3 ++ .../carbondata/presto/CarbondataRecordSet.java | 6 +-- .../presto/CarbondataRecordSetProvider.java | 14 +------ .../presto/CarbondataSplitManager.java | 26 ++++++++----- .../presto/impl/CarbonTableCacheModel.java | 4 ++ .../presto/impl/CarbonTableConfig.java | 3 ++ .../presto/impl/CarbonTableReader.java | 40 +++++++++----------- 11 files changed, 55 insertions(+), 61 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/10bddd88/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataColumnConstraint.java ---------------------------------------------------------------------- diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataColumnConstraint.java b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataColumnConstraint.java index 0665aed..82c7c78 100755 --- a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataColumnConstraint.java +++ b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataColumnConstraint.java @@ -29,6 +29,9 @@ import java.util.Optional; import static com.google.common.base.Objects.toStringHelper; import static java.util.Objects.requireNonNull; +/** + * Encapsulating presto Tuple-domain + */ public class CarbondataColumnConstraint { private final String name; private final boolean invertedindexed; http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/10bddd88/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataColumnHandle.java ---------------------------------------------------------------------- diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataColumnHandle.java b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataColumnHandle.java index cc10165..b9152b5 100755 --- a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataColumnHandle.java +++ b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataColumnHandle.java @@ -68,12 +68,6 @@ public class CarbondataColumnHandle implements ColumnHandle { return columnUniqueId; } - /** - * ordinalPosition of a columnhandle is the -> number of the column in the entire list of columns of this table - * IT DOESNT DEPEND ON THE QUERY (select clm3, clm0, clm1 from tablename) - * The columnhandle of clm3 : has ordinalposition = 3 - */ - @JsonCreator public CarbondataColumnHandle(@JsonProperty("connectorId") String connectorId, @JsonProperty("columnName") String columnName, @JsonProperty("columnType") Type columnType, @JsonProperty("ordinalPosition") int ordinalPosition, http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/10bddd88/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataConnectorFactory.java ---------------------------------------------------------------------- diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataConnectorFactory.java b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataConnectorFactory.java index b146a2e..66c007d 100755 --- a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataConnectorFactory.java +++ b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataConnectorFactory.java @@ -31,6 +31,10 @@ import java.util.Map; import static java.util.Objects.requireNonNull; +/** + * Build Carbondata Connector + * It will be called by CarbondataPlugin + */ public class CarbondataConnectorFactory implements ConnectorFactory { private final String name; http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/10bddd88/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataMetadata.java ---------------------------------------------------------------------- diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataMetadata.java b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataMetadata.java index d2c5ab6..d938a3d 100755 --- a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataMetadata.java +++ b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataMetadata.java @@ -69,13 +69,6 @@ public class CarbondataMetadata implements ConnectorMetadata { @Override public List<SchemaTableName> listTables(ConnectorSession session, String schemaNameOrNull) { - /*List<SchemaTableName> all = carbonTableReader.getTableList(); - if(schemaNameOrNull != null) - { - return all.stream().filter(a -> schemaNameOrNull.equals(a.getSchemaName())).collect(Collectors.toList()); - } - return all;*/ - List<String> schemaNames; if (schemaNameOrNull != null) { schemaNames = ImmutableList.of(schemaNameOrNull); http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/10bddd88/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataRecordCursor.java ---------------------------------------------------------------------- diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataRecordCursor.java b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataRecordCursor.java index 8eef1c9..ad47f75 100755 --- a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataRecordCursor.java +++ b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataRecordCursor.java @@ -78,6 +78,9 @@ public class CarbondataRecordCursor implements RecordCursor { return columnHandles.get(field).getColumnType(); } + /** + * get next Row/Page + */ @Override public boolean advanceNextPosition() { if (nanoStart == 0) { http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/10bddd88/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataRecordSet.java ---------------------------------------------------------------------- diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataRecordSet.java b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataRecordSet.java index af37728..d3fd7a0 100755 --- a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataRecordSet.java +++ b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataRecordSet.java @@ -74,12 +74,12 @@ public class CarbondataRecordSet implements RecordSet { return columns.stream().map(a -> a.getColumnType()).collect(Collectors.toList()); } + /** + * get data blocks via Carbondata QueryModel API + */ @Override public RecordCursor cursor() { List<TableBlockInfo> tableBlockInfoList = new ArrayList<TableBlockInfo>(); - //tableBlockInfoList.add(split.getLocalInputSplit().getTableBlockInfo()); - /*BlockletInfos blockletInfos = new BlockletInfos(split.getLocalInputSplit().getNumberOfBlocklets(), 0, - split.getLocalInputSplit().getNumberOfBlocklets());*/ tableBlockInfoList.add(new TableBlockInfo(split.getLocalInputSplit().getPath().toString(), split.getLocalInputSplit().getStart(), split.getLocalInputSplit().getSegmentId(), split.getLocalInputSplit().getLocations().toArray(new String[0]), http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/10bddd88/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataRecordSetProvider.java ---------------------------------------------------------------------- diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataRecordSetProvider.java b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataRecordSetProvider.java index 63b926f..8b087df 100755 --- a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataRecordSetProvider.java +++ b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataRecordSetProvider.java @@ -72,7 +72,6 @@ public class CarbondataRecordSetProvider implements ConnectorRecordSetProvider { requireNonNull(split, "split is null"); requireNonNull(columns, "columns is null"); - // Convert split CarbondataSplit cdSplit = checkType(split, CarbondataSplit.class, "split is not class CarbondataSplit"); checkArgument(cdSplit.getConnectorId().equals(connectorId), "split is not for this connector"); @@ -111,11 +110,11 @@ public class CarbondataRecordSetProvider implements ConnectorRecordSetProvider { fillFilter2QueryModel(queryModel, cdSplit.getConstraints(), targetTable); // Return new record set - return new CarbondataRecordSet(targetTable,/*connector,*/ session, /*config, */cdSplit, + return new CarbondataRecordSet(targetTable, session, cdSplit, handles.build(), queryModel); } - // Build filter for QueryModel (copy from CarbonInputFormat=> createRecordReader) + // Build filter for QueryModel private void fillFilter2QueryModel(QueryModel queryModel, TupleDomain<ColumnHandle> originalConstraint, CarbonTable carbonTable) { @@ -139,14 +138,9 @@ public class CarbondataRecordSetProvider implements ConnectorRecordSetProvider { checkArgument(domain.getType().isOrderable(), "Domain type must be orderable"); if (domain.getValues().isNone()) { - //return QueryBuilders.filteredQuery(null, FilterBuilders.missingFilter(columnName)); - //return domain.isNullAllowed() ? columnName + " IS NULL" : "FALSE"; - //new Expression() } if (domain.getValues().isAll()) { - //return QueryBuilders.filteredQuery(null, FilterBuilders.existsFilter(columnName)); - //return domain.isNullAllowed() ? "TRUE" : columnName + " IS NOT NULL"; } List<Object> singleValues = new ArrayList<>(); @@ -166,7 +160,6 @@ public class CarbondataRecordSetProvider implements ConnectorRecordSetProvider { } else { GreaterThanExpression greater = new GreaterThanExpression(colExpression, new LiteralExpression(value, coltype)); - //greater.setRangeExpression(true); rangeFilter.add(greater); } break; @@ -174,7 +167,6 @@ public class CarbondataRecordSetProvider implements ConnectorRecordSetProvider { GreaterThanEqualToExpression greater = new GreaterThanEqualToExpression(colExpression, new LiteralExpression(value, coltype)); - //greater.setRangeExpression(true); rangeFilter.add(greater); break; case BELOW: @@ -191,13 +183,11 @@ public class CarbondataRecordSetProvider implements ConnectorRecordSetProvider { case EXACTLY: LessThanEqualToExpression less = new LessThanEqualToExpression(colExpression, new LiteralExpression(value, coltype)); - //less.setRangeExpression(true); rangeFilter.add(less); break; case BELOW: LessThanExpression less2 = new LessThanExpression(colExpression, new LiteralExpression(value, coltype)); - //less2.setRangeExpression(true); rangeFilter.add(less2); break; default: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/10bddd88/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataSplitManager.java ---------------------------------------------------------------------- diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataSplitManager.java b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataSplitManager.java index a8902eb..f3efb36 100755 --- a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataSplitManager.java +++ b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataSplitManager.java @@ -50,6 +50,10 @@ import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkState; import static java.util.Objects.requireNonNull; +/** + * Build Carbontable splits + * filtering irrelevant blocks + */ public class CarbondataSplitManager implements ConnectorSplitManager { private final String connectorId; @@ -67,7 +71,7 @@ public class CarbondataSplitManager implements ConnectorSplitManager { CarbondataTableHandle tableHandle = layoutHandle.getTable(); SchemaTableName key = tableHandle.getSchemaTableName(); - //get all filter domain + // Packaging presto-TupleDomain into CarbondataColumnConstraint, to decouple from presto-spi Module List<CarbondataColumnConstraint> rebuildConstraints = getColumnConstraints(layoutHandle.getConstraint()); @@ -106,6 +110,12 @@ public class CarbondataSplitManager implements ConnectorSplitManager { return constraintBuilder.build(); } + /** + * Convert presto-TupleDomain predication into Carbon scan express condition + * @param originalConstraint presto-TupleDomain + * @param carbonTable + * @return + */ public Expression parseFilterExpression(TupleDomain<ColumnHandle> originalConstraint, CarbonTable carbonTable) { ImmutableList.Builder<Expression> filters = ImmutableList.builder(); @@ -136,14 +146,9 @@ public class CarbondataSplitManager implements ConnectorSplitManager { checkArgument(domain.getType().isOrderable(), "Domain type must be orderable"); if (domain.getValues().isNone()) { - //return QueryBuilders.filteredQuery(null, FilterBuilders.missingFilter(columnName)); - //return domain.isNullAllowed() ? columnName + " IS NULL" : "FALSE"; - //new Expression() } if (domain.getValues().isAll()) { - //return QueryBuilders.filteredQuery(null, FilterBuilders.existsFilter(columnName)); - //return domain.isNullAllowed() ? "TRUE" : columnName + " IS NOT NULL"; } List<Object> singleValues = new ArrayList<>(); @@ -163,7 +168,6 @@ public class CarbondataSplitManager implements ConnectorSplitManager { } else { GreaterThanExpression greater = new GreaterThanExpression(colExpression, new LiteralExpression(value, coltype)); - //greater.setRangeExpression(true); rangeFilter.add(greater); } break; @@ -171,7 +175,6 @@ public class CarbondataSplitManager implements ConnectorSplitManager { GreaterThanEqualToExpression greater = new GreaterThanEqualToExpression(colExpression, new LiteralExpression(value, coltype)); - //greater.setRangeExpression(true); rangeFilter.add(greater); break; case BELOW: @@ -188,13 +191,11 @@ public class CarbondataSplitManager implements ConnectorSplitManager { case EXACTLY: LessThanEqualToExpression less = new LessThanEqualToExpression(colExpression, new LiteralExpression(value, coltype)); - //less.setRangeExpression(true); rangeFilter.add(less); break; case BELOW: LessThanExpression less2 = new LessThanExpression(colExpression, new LiteralExpression(value, coltype)); - //less2.setRangeExpression(true); rangeFilter.add(less2); break; default: @@ -249,6 +250,11 @@ public class CarbondataSplitManager implements ConnectorSplitManager { return finalFilters; } + /** + * Convert presto spi Type into Carbondata Type + * @param colType + * @return + */ public static DataType Spi2CarbondataTypeMapper(Type colType) { if (colType == BooleanType.BOOLEAN) return DataType.BOOLEAN; else if (colType == SmallintType.SMALLINT) return DataType.SHORT; http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/10bddd88/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableCacheModel.java ---------------------------------------------------------------------- diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableCacheModel.java b/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableCacheModel.java index b138f18..45755d1 100755 --- a/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableCacheModel.java +++ b/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableCacheModel.java @@ -22,6 +22,10 @@ import org.apache.carbondata.core.metadata.schema.table.CarbonTable; import org.apache.carbondata.core.metadata.schema.table.TableInfo; import org.apache.carbondata.core.util.path.CarbonTablePath; +/** + * Caching Carbon meta(e.g. TableIdentifier, TablePath, TableInfo, CarbonTable) in Class CarbonTableReader + * to speed up query + */ public class CarbonTableCacheModel { public CarbonTableIdentifier carbonTableIdentifier; http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/10bddd88/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableConfig.java ---------------------------------------------------------------------- diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableConfig.java b/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableConfig.java index a0ef63f..677cefd 100755 --- a/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableConfig.java +++ b/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableConfig.java @@ -21,6 +21,9 @@ import io.airlift.configuration.Config; import javax.validation.constraints.NotNull; +/** + * Configuration read from etc/catalog/carbondata.properties + */ public class CarbonTableConfig { //read from config http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/10bddd88/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableReader.java ---------------------------------------------------------------------- diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableReader.java b/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableReader.java index 14ecfbc..bb482b0 100755 --- a/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableReader.java +++ b/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableReader.java @@ -71,22 +71,21 @@ import java.util.stream.Stream; import static java.util.Objects.requireNonNull; +/** CarbonTableReader will be a facade of these utils + * + * 1:CarbonMetadata,(logic table) + * 2:FileFactory, (physic table file) + * 3:CarbonCommonFactory, (offer some ) + * 4:DictionaryFactory, (parse dictionary util) + */ public class CarbonTableReader { - /** CarbonTableReader will be a facade of these utils - * - * 1:CarbonMetadata,(logic table) - * 2:FileFactory, (physic table file) - * 3:CarbonCommonFactory, (offer some ) - * 4:DictionaryFactory, (parse dictionary util) - */ - private CarbonTableConfig config; private List<SchemaTableName> tableList; private CarbonFile dbStore; private FileFactory.FileType fileType; - //as a cache for Carbon reader + // A cache for Carbon reader private ConcurrentHashMap<SchemaTableName, CarbonTableCacheModel> cc; @Inject public CarbonTableReader(CarbonTableConfig config) { @@ -94,7 +93,7 @@ public class CarbonTableReader { this.cc = new ConcurrentHashMap<>(); } - //for worker node to initialize carbon metastore + // for worker node to initialize carbon metastore public CarbonTableCacheModel getCarbonCache(SchemaTableName table) { if (!cc.containsKey(table)) { try (ThreadContextClassLoader ignored = new ThreadContextClassLoader( @@ -120,7 +119,7 @@ public class CarbonTableReader { return updateSchemaList(); } - //default PathFilter + // default PathFilter private static final PathFilter DefaultFilter = new PathFilter() { @Override public boolean accept(Path path) { return CarbonTablePath.isCarbonDataFile(path.getName()); @@ -177,7 +176,7 @@ public class CarbonTableReader { } public void updateSchemaTables() { - //update logic determine later + // update logic determine later if (dbStore == null) { updateSchemaList(); } @@ -232,14 +231,14 @@ public class CarbonTableReader { (org.apache.carbondata.format.TableInfo) thriftReader.read(); thriftReader.close(); - //Format Level TableInfoï¼ need transfer to Code Level TableInfo + // Step3: Transform Format Level TableInfo to Code Level TableInfo SchemaConverter schemaConverter = new ThriftWrapperSchemaConverterImpl(); TableInfo wrapperTableInfo = schemaConverter .fromExternalToWrapperTableInfo(tableInfo, table.getSchemaName(), table.getTableName(), storePath); wrapperTableInfo.setMetaDataFilepath( CarbonTablePath.getFolderContainingFile(cache.carbonTablePath.getSchemaFilePath())); - //load metadata info into CarbonMetadata + // Step4: Load metadata info into CarbonMetadata CarbonMetadata.getInstance().loadTableMetadata(wrapperTableInfo); cache.tableInfo = wrapperTableInfo; @@ -256,7 +255,7 @@ public class CarbonTableReader { public List<CarbonLocalInputSplit> getInputSplits2(CarbonTableCacheModel tableCacheModel, Expression filters) throws Exception { - //å¤çfilter, ä¸æ¨filterï¼å°åºç¨å¨Segmentçç´¢å¼ä¸ + // need apply filters to segment FilterExpressionProcessor filterExpressionProcessor = new FilterExpressionProcessor(); AbsoluteTableIdentifier absoluteTableIdentifier = @@ -297,7 +296,7 @@ public class CarbonTableReader { .resolveFilter(filters, tableCacheModel.carbonTable.getAbsoluteTableIdentifier()); List<CarbonLocalInputSplit> result = new ArrayList<>(); - //for each segment fetch blocks matching filter in Driver BTree + // for each segment fetch blocks matching filter in Driver BTree for (String segmentNo : tableCacheModel.segments) { try { List<DataRefNode> dataRefNodes = @@ -338,7 +337,7 @@ public class CarbonTableReader { //DriverQueryStatisticsRecorder recorder = CarbonTimeStatisticsFactory.getQueryStatisticsRecorderInstance(); //QueryStatistic statistic = new QueryStatistic(); - //读åSegment å é¨çIndex + // read segment index Map<SegmentTaskIndexStore.TaskBucketHolder, AbstractIndex> segmentIndexMap = getSegmentAbstractIndexs(absoluteTableIdentifier, tablePath, segmentId, cacheClient, updateStatusManager); @@ -349,13 +348,10 @@ public class CarbonTableReader { // build result for (AbstractIndex abstractIndex : segmentIndexMap.values()) { List<DataRefNode> filterredBlocks; - // if no filter is given get all blocks from Btree Index + // if no filter is given, get all blocks from Btree Index if (null == resolver) { filterredBlocks = getDataBlocksOfIndex(abstractIndex); } else { - //ignore filter - //filterredBlocks = getDataBlocksOfIndex(abstractIndex); - // apply filter and get matching blocks filterredBlocks = filterExpressionProcessor .getFilterredBlocks(abstractIndex.getDataRefNode(), resolver, abstractIndex, @@ -568,12 +564,10 @@ public class CarbonTableReader { try { Configuration conf = new Configuration(); fs = segmentPath.getFileSystem(conf); - //fs.initialize(segmentPath.toUri(), conf); RemoteIterator<LocatedFileStatus> iter = fs.listLocatedStatus(segmentPath); while (iter.hasNext()) { LocatedFileStatus stat = iter.next(); - //if(stat.getPath().toString().contains("carbondata"))//åçcarbondataçcarbonInputFilterçå®ç° if (DefaultFilter.accept(stat.getPath())) { if (stat.isDirectory()) { addInputPathRecursively(result, fs, stat.getPath(), DefaultFilter);
