Repository: carbondata Updated Branches: refs/heads/master 2d24e18b8 -> 110f9b21b
[CARBONDATA-1397]Fix findbugs issues in carbondata-presto module Fix findbugs issues in carbondata-presto module This closes #1271 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/110f9b21 Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/110f9b21 Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/110f9b21 Branch: refs/heads/master Commit: 110f9b21bddfe28a1c3a6b1700623addd000f2c0 Parents: 2d24e18 Author: Raghunandan S <[email protected]> Authored: Sat Aug 19 17:23:33 2017 +0530 Committer: Jacky Li <[email protected]> Committed: Sun Aug 20 02:29:52 2017 +0800 ---------------------------------------------------------------------- .../carbondata/presto/CarbondataMetadata.java | 12 +++---- .../carbondata/presto/CarbondataPageSource.java | 4 +-- .../carbondata/presto/CarbondataRecordSet.java | 7 ---- .../presto/CarbondataRecordSetProvider.java | 21 ++++++------ .../presto/CarbondataSplitManager.java | 34 +++++++++----------- .../presto/impl/CarbonTableReader.java | 8 +---- 6 files changed, 33 insertions(+), 53 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/carbondata/blob/110f9b21/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataMetadata.java ---------------------------------------------------------------------- diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataMetadata.java b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataMetadata.java index a132fb4..a81c06f 100755 --- a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataMetadata.java +++ b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataMetadata.java @@ -120,7 +120,7 @@ public class CarbondataMetadata implements ConnectorMetadata { List<CarbonColumn> carbonColumns = carbonTable.getCreateOrderColumn(schemaTableName.getTableName()); for (CarbonColumn col : carbonColumns) { //show columns command will return these data - Type columnType = CarbondataType2SpiMapper(col.getColumnSchema()); + Type columnType = carbonDataType2SpiMapper(col.getColumnSchema()); ColumnMetadata columnMeta = new ColumnMetadata(col.getColumnSchema().getColumnName(), columnType); columnsMetaList.add(columnMeta); } @@ -151,11 +151,7 @@ public class CarbondataMetadata implements ConnectorMetadata { for (CarbonDimension column : cb.getDimensionByTableName(tableName)) { ColumnSchema cs = column.getColumnSchema(); - int complex = column.getComplexTypeOrdinal(); - column.getNumberOfChild(); - column.getListOfChildDimensions(); - - Type spiType = CarbondataType2SpiMapper(cs); + Type spiType = carbonDataType2SpiMapper(cs); columnHandles.put(cs.getColumnName(), new CarbondataColumnHandle(connectorId, cs.getColumnName(), spiType, column.getSchemaOrdinal(), column.getKeyOrdinal(), column.getColumnGroupOrdinal(), false, cs.getColumnGroupId(), @@ -165,7 +161,7 @@ public class CarbondataMetadata implements ConnectorMetadata { for (CarbonMeasure measure : cb.getMeasureByTableName(tableName)) { ColumnSchema cs = measure.getColumnSchema(); - Type spiType = CarbondataType2SpiMapper(cs); + Type spiType = carbonDataType2SpiMapper(cs); columnHandles.put(cs.getColumnName(), new CarbondataColumnHandle(connectorId, cs.getColumnName(), spiType, cs.getSchemaOrdinal(), measure.getOrdinal(), cs.getColumnGroupId(), true, cs.getColumnGroupId(), @@ -223,7 +219,7 @@ public class CarbondataMetadata implements ConnectorMetadata { return getTableMetadata(carbondataTableHandle.getSchemaTableName()); } - public static Type CarbondataType2SpiMapper(ColumnSchema columnSchema) { + public static Type carbonDataType2SpiMapper(ColumnSchema columnSchema) { DataType colType = columnSchema.getDataType(); switch (colType) { case BOOLEAN: http://git-wip-us.apache.org/repos/asf/carbondata/blob/110f9b21/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataPageSource.java ---------------------------------------------------------------------- diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataPageSource.java b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataPageSource.java index c03983e..f7f6d1e 100644 --- a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataPageSource.java +++ b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataPageSource.java @@ -190,14 +190,12 @@ public class CarbondataPageSource implements ConnectorPageSource { /** * Using the LazyBlockLoader */ - private final class CarbonBlockLoader implements LazyBlockLoader<LazyBlock> { + private static final class CarbonBlockLoader implements LazyBlockLoader<LazyBlock> { private boolean loaded; private Block dataBlock; - private Type type; public CarbonBlockLoader(Block dataBlock, Type type) { this.dataBlock = dataBlock; - this.type = type; } @Override public void load(LazyBlock block) { http://git-wip-us.apache.org/repos/asf/carbondata/blob/110f9b21/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataRecordSet.java ---------------------------------------------------------------------- diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataRecordSet.java b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataRecordSet.java index 661e83f..435b008 100755 --- a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataRecordSet.java +++ b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataRecordSet.java @@ -49,10 +49,6 @@ import static org.apache.carbondata.presto.Types.checkType; public class CarbondataRecordSet implements RecordSet { - private CarbonTable carbonTable; - private TupleDomain<ColumnHandle> originalConstraint; - private Expression carbonConstraint; - private List<CarbondataColumnConstraint> rebuildConstraints; private QueryModel queryModel; private CarbondataSplit split; private List<CarbondataColumnHandle> columns; @@ -62,10 +58,7 @@ public class CarbondataRecordSet implements RecordSet { public CarbondataRecordSet(CarbonTable carbonTable, ConnectorSession session, ConnectorSplit split, List<CarbondataColumnHandle> columns, QueryModel queryModel) { - this.carbonTable = carbonTable; this.split = checkType(split, CarbondataSplit.class, "connectorSplit"); - this.originalConstraint = this.split.getConstraints(); - this.rebuildConstraints = this.split.getRebuildConstraints(); this.queryModel = queryModel; this.columns = columns; this.readSupport = new CarbonDictionaryDecodeReaderSupport(); http://git-wip-us.apache.org/repos/asf/carbondata/blob/110f9b21/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataRecordSetProvider.java ---------------------------------------------------------------------- diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataRecordSetProvider.java b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataRecordSetProvider.java index 532ab87..3385170 100755 --- a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataRecordSetProvider.java +++ b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataRecordSetProvider.java @@ -75,17 +75,18 @@ public class CarbondataRecordSetProvider implements ConnectorRecordSetProvider { checkType(split, CarbondataSplit.class, "split is not class CarbondataSplit"); checkArgument(carbondataSplit.getConnectorId().equals(connectorId), "split is not for this connector"); + StringBuffer targetColsBuffer = new StringBuffer(); String targetCols = ""; // Convert all columns handles ImmutableList.Builder<CarbondataColumnHandle> handles = ImmutableList.builder(); for (ColumnHandle handle : columns) { handles.add(checkType(handle, CarbondataColumnHandle.class, "handle")); - targetCols += ((CarbondataColumnHandle) handle).getColumnName() + ","; + targetColsBuffer.append(((CarbondataColumnHandle) handle).getColumnName()).append(","); } // Build column projection(check the column order) - if (targetCols.length() > 0) { - targetCols = targetCols.substring(0, targetCols.length() - 1); + if (targetColsBuffer.length() > 0) { + targetCols = targetColsBuffer.substring(0, targetCols.length() - 1); } else { @@ -131,7 +132,7 @@ public class CarbondataRecordSetProvider implements ConnectorRecordSetProvider { CarbondataColumnHandle cdch = (CarbondataColumnHandle) c; Type type = cdch.getColumnType(); - DataType coltype = Spi2CarbondataTypeMapper(cdch); + DataType coltype = spi2CarbondataTypeMapper(cdch); Expression colExpression = new ColumnExpression(cdch.getColumnName(), coltype); domain = originalConstraint.getDomains().get().get(c); @@ -151,7 +152,7 @@ public class CarbondataRecordSetProvider implements ConnectorRecordSetProvider { } else { List<Expression> rangeConjuncts = new ArrayList<>(); if (!range.getLow().isLowerUnbounded()) { - Object value = ConvertDataByType(range.getLow().getValue(), type); + Object value = convertDataByType(range.getLow().getValue(), type); switch (range.getLow().getBound()) { case ABOVE: if (type == TimestampType.TIMESTAMP) { @@ -175,7 +176,7 @@ public class CarbondataRecordSetProvider implements ConnectorRecordSetProvider { } } if (!range.getHigh().isUpperUnbounded()) { - Object value = ConvertDataByType(range.getHigh().getValue(), type); + Object value = convertDataByType(range.getHigh().getValue(), type); switch (range.getHigh().getBound()) { case ABOVE: throw new IllegalArgumentException("High marker should never use ABOVE bound"); @@ -211,7 +212,7 @@ public class CarbondataRecordSetProvider implements ConnectorRecordSetProvider { } else if (singleValues.size() > 1) { ListExpression candidates = null; List<Expression> exs = singleValues.stream().map((a) -> { - return new LiteralExpression(ConvertDataByType(a, type), coltype); + return new LiteralExpression(convertDataByType(a, type), coltype); }).collect(Collectors.toList()); candidates = new ListExpression(exs); @@ -246,7 +247,7 @@ public class CarbondataRecordSetProvider implements ConnectorRecordSetProvider { CarbonInputFormatUtil.resolveFilter(finalFilters, queryModel.getAbsoluteTableIdentifier())); } - public static DataType Spi2CarbondataTypeMapper(CarbondataColumnHandle carbondataColumnHandle) { + public static DataType spi2CarbondataTypeMapper(CarbondataColumnHandle carbondataColumnHandle) { Type colType = carbondataColumnHandle.getColumnType(); if (colType == BooleanType.BOOLEAN) return DataType.BOOLEAN; else if (colType == SmallintType.SMALLINT) return DataType.SHORT; @@ -261,8 +262,8 @@ public class CarbondataRecordSetProvider implements ConnectorRecordSetProvider { else return DataType.STRING; } - public Object ConvertDataByType(Object rawdata, Type type) { - if (type.equals(IntegerType.INTEGER)) return new Integer((rawdata.toString())); + public Object convertDataByType(Object rawdata, Type type) { + if (type.equals(IntegerType.INTEGER)) return Integer.valueOf(rawdata.toString()); else if (type.equals(BigintType.BIGINT)) return (Long) rawdata; else if (type.equals(VarcharType.VARCHAR)) return ((Slice) rawdata).toStringUtf8(); else if (type.equals(BooleanType.BOOLEAN)) return (Boolean) (rawdata); http://git-wip-us.apache.org/repos/asf/carbondata/blob/110f9b21/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataSplitManager.java ---------------------------------------------------------------------- diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataSplitManager.java b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataSplitManager.java index 0ce0600..cf34f1d 100755 --- a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataSplitManager.java +++ b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataSplitManager.java @@ -99,19 +99,17 @@ public class CarbondataSplitManager implements ConnectorSplitManager { CarbonTableCacheModel cache = carbonTableReader.getCarbonCache(key); Expression filters = parseFilterExpression(layoutHandle.getConstraint(), cache.carbonTable); - if (cache != null) { - try { - List<CarbonLocalInputSplit> splits = carbonTableReader.getInputSplits2(cache, filters); - - ImmutableList.Builder<ConnectorSplit> cSplits = ImmutableList.builder(); - for (CarbonLocalInputSplit split : splits) { - cSplits.add(new CarbondataSplit(connectorId, tableHandle.getSchemaTableName(), - layoutHandle.getConstraint(), split, rebuildConstraints)); - } - return new FixedSplitSource(cSplits.build()); - } catch (Exception ex) { - System.out.println(ex.toString()); + try { + List<CarbonLocalInputSplit> splits = carbonTableReader.getInputSplits2(cache, filters); + + ImmutableList.Builder<ConnectorSplit> cSplits = ImmutableList.builder(); + for (CarbonLocalInputSplit split : splits) { + cSplits.add(new CarbondataSplit(connectorId, tableHandle.getSchemaTableName(), + layoutHandle.getConstraint(), split, rebuildConstraints)); } + return new FixedSplitSource(cSplits.build()); + } catch (Exception ex) { + System.out.println(ex.toString()); } return null; } @@ -182,7 +180,7 @@ public class CarbondataSplitManager implements ConnectorSplitManager { } else { List<Expression> rangeConjuncts = new ArrayList<>(); if (!range.getLow().isLowerUnbounded()) { - Object value = ConvertDataByType(range.getLow().getValue(), type); + Object value = convertDataByType(range.getLow().getValue(), type); switch (range.getLow().getBound()) { case ABOVE: if (type == TimestampType.TIMESTAMP) { @@ -206,7 +204,7 @@ public class CarbondataSplitManager implements ConnectorSplitManager { } } if (!range.getHigh().isUpperUnbounded()) { - Object value = ConvertDataByType(range.getHigh().getValue(), type); + Object value = convertDataByType(range.getHigh().getValue(), type); switch (range.getHigh().getBound()) { case ABOVE: throw new IllegalArgumentException("High marker should never use ABOVE bound"); @@ -239,7 +237,7 @@ public class CarbondataSplitManager implements ConnectorSplitManager { } else if (singleValues.size() > 1) { ListExpression candidates = null; List<Expression> exs = singleValues.stream().map((a) -> { - return new LiteralExpression(ConvertDataByType(a, type), coltype); + return new LiteralExpression(convertDataByType(a, type), coltype); }).collect(Collectors.toList()); candidates = new ListExpression(exs); @@ -279,7 +277,7 @@ public class CarbondataSplitManager implements ConnectorSplitManager { * @param colType * @return */ - public static DataType Spi2CarbondataTypeMapper(Type colType) { + public static DataType spi2CarbondataTypeMapper(Type colType) { if (colType == BooleanType.BOOLEAN) return DataType.BOOLEAN; else if (colType == SmallintType.SMALLINT) return DataType.SHORT; else if (colType == IntegerType.INTEGER) return DataType.INT; @@ -292,8 +290,8 @@ public class CarbondataSplitManager implements ConnectorSplitManager { else return DataType.STRING; } - public Object ConvertDataByType(Object rawdata, Type type) { - if (type.equals(IntegerType.INTEGER)) return new Integer((rawdata.toString())); + public Object convertDataByType(Object rawdata, Type type) { + if (type.equals(IntegerType.INTEGER)) return Integer.valueOf(rawdata.toString()); else if (type.equals(BigintType.BIGINT)) return (Long) rawdata; else if (type.equals(VarcharType.VARCHAR)) return ((Slice) rawdata).toStringUtf8(); else if (type.equals(BooleanType.BOOLEAN)) return (Boolean) (rawdata); http://git-wip-us.apache.org/repos/asf/carbondata/blob/110f9b21/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableReader.java ---------------------------------------------------------------------- diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableReader.java b/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableReader.java index 8aa18d5..6ddb8be 100755 --- a/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableReader.java +++ b/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableReader.java @@ -361,7 +361,6 @@ public class CarbonTableReader { tableCacheModel.carbonTable.getAbsoluteTableIdentifier(); CacheClient cacheClient = new CacheClient(absoluteTableIdentifier.getStorePath()); List<String> invalidSegments = new ArrayList<>(); - List<UpdateVO> invalidTimestampsList = new ArrayList<>(); // get all valid segments and set them into the configuration SegmentUpdateStatusManager updateStatusManager = @@ -377,9 +376,6 @@ public class CarbonTableReader { // remove entry in the segment index if there are invalid segments invalidSegments.addAll(segments.getInvalidSegments()); - for (String invalidSegmentId : invalidSegments) { - invalidTimestampsList.add(updateStatusManager.getInvalidTimestampRange(invalidSegmentId)); - } if (invalidSegments.size() > 0) { List<TableSegmentUniqueIdentifier> invalidSegmentsIds = new ArrayList<>(invalidSegments.size()); @@ -536,8 +532,6 @@ public class CarbonTableReader { if (segmentIndexMap == null || isSegmentUpdated) { List<FileStatus> fileStatusList = new LinkedList<FileStatus>(); - List<String> segs = new ArrayList<>(); - segs.add(segmentId); FileSystem fs = getFileStatusOfSegments(new String[] { segmentId }, tablePath, fileStatusList); @@ -556,7 +550,7 @@ public class CarbonTableReader { List<TableBlockInfo> tableBlockInfoList = new ArrayList<>(); for (FileSplit inputSplit : carbonSplits) { - if ((null == updateDetails) || ((null != updateDetails) && isValidBlockBasedOnUpdateDetails( + if ((null == updateDetails) || (isValidBlockBasedOnUpdateDetails( taskKeys, inputSplit, updateDetails, updateStatusManager, segmentId))) { BlockletInfos blockletInfos = new BlockletInfos(0, 0,
