[carbondata] branch master updated: [CARBONDATA-3889] Cleanup duplicated code in carbondata-core module
This is an automated email from the ASF dual-hosted git repository.
manhua pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/carbondata.git
The following commit(s) were added to refs/heads/master by this push:
new 0367516 [CARBONDATA-3889] Cleanup duplicated code in carbondata-core
module
0367516 is described below
commit 03675164a0065d7e5f9e584e5556bb74b36c0461
Author: QiangCai
AuthorDate: Thu Jul 23 17:25:45 2020 +0800
[CARBONDATA-3889] Cleanup duplicated code in carbondata-core module
Why is this PR needed?
There are duplicated code in carbondata-core module
What changes were proposed in this PR?
1.Cleanup duplicated code in carbondata-core module
2.Fix the typo
Does this PR introduce any user interface change?
No
Is any new testcase added?
No
This closes #3860
---
.../core/datastore/block/SegmentProperties.java| 8 +-
.../chunk/impl/FixedLengthDimensionColumnPage.java | 57 --
.../safe/SafeAbstractDimensionDataChunkStore.java | 6 +-
.../datastore/filesystem/AlluxioCarbonFile.java| 2 +-
.../core/datastore/impl/FileFactory.java | 18 +---
.../page/encoding/DefaultEncodingFactory.java | 12 +--
.../apache/carbondata/core/index/IndexUtil.java| 27 +
.../core/index/dev/expr/AndIndexExprWrapper.java | 35 ++
.../core/indexstore/BlockletIndexStore.java| 2 +-
.../core/indexstore/UnsafeMemoryDMStore.java | 2 +-
.../core/indexstore/blockletindex/BlockIndex.java | 2 +-
.../core/indexstore/schema/SchemaGenerator.java| 58 --
...java => AbstractDirectDictionaryGenerator.java} | 101 --
.../timestamp/DateDirectDictionaryGenerator.java | 90 +---
.../TimeStampDirectDictionaryGenerator.java| 95 +
.../carbondata/core/keygenerator/mdkey/Bits.java | 80 --
.../ThriftWrapperSchemaConverterImpl.java | 4 +-
.../core/reader/CarbonDeleteFilesDataReader.java | 46
.../scan/filter/FilterExpressionProcessor.java | 54 +-
.../carbondata/core/scan/filter/FilterUtil.java| 2 +-
.../filter/executer/AndFilterExecutorImpl.java | 4 +-
.../filter/executer/ExcludeFilterExecutorImpl.java | 34 +++---
.../filter/executer/IncludeFilterExecutorImpl.java | 8 +-
.../executer/MeasureColumnExecutorFilterInfo.java | 2 +-
.../scan/filter/resolver/FilterResolverIntf.java | 4 +-
.../resolver/RowLevelFilterResolverImpl.java | 2 +-
.../resolver/RowLevelRangeFilterResolverImpl.java | 4 +-
.../resolverinfo/DimColumnResolvedFilterInfo.java | 4 +-
.../resolverinfo/TrueConditionalResolverImpl.java | 4 +-
...apperDirectWithDeleteDeltaAndInvertedIndex.java | 2 +-
.../scan/scanner/impl/BlockletFilterScanner.java | 49 -
.../core/segmentmeta/SegmentMetaDataInfoStats.java | 25 +
.../core/util/AbstractDataFileFooterConverter.java | 117 +++--
.../core/util/CarbonLoadStatisticsImpl.java| 101 +-
.../carbondata/core/util/CarbonProperties.java | 25 ++---
.../core/util/DataFileFooterConverter.java | 15 +--
.../core/util/DataFileFooterConverterV3.java | 14 +--
.../apache/carbondata/core/util/DataTypeUtil.java | 111 ---
.../core/writer/CarbonIndexFileMergeWriter.java| 37 +++
.../core/keygenerator/mdkey/BitsUnitTest.java | 49 -
40 files changed, 248 insertions(+), 1064 deletions(-)
diff --git
a/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentProperties.java
b/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentProperties.java
index fe28a37..2b8a049 100644
---
a/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentProperties.java
+++
b/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentProperties.java
@@ -256,11 +256,11 @@ public class SegmentProperties {
*/
protected long getFingerPrinter() {
if (this.fingerPrinter == Long.MAX_VALUE) {
- long dimensionsFingerPrinter = getFingerprinter(this.dimensions.stream()
+ long dimensionsFingerPrinter = getFingerPrinter(this.dimensions.stream()
.map(t -> t.getColumnSchema()).collect(Collectors.toList()));
- long measuresFingerPrinter = getFingerprinter(this.measures.stream()
+ long measuresFingerPrinter = getFingerPrinter(this.measures.stream()
.map(t -> t.getColumnSchema()).collect(Collectors.toList()));
- long complexFingerPrinter =
getFingerprinter(this.complexDimensions.stream()
+ long complexFingerPrinter =
getFingerPrinter(this.complexDimensions.stream()
.map(t -> t.getColumnSchema()).collect(Collectors.toList()));
this.fingerPrinter = (dimensionsFingerPrinter >>
DIMENSIONS_FINGER_PRINTER_SHIFT)
^ (measuresFingerPrinter >> MEASU
[carbondata] branch master updated: [CARBONDATA-3889] Cleanup code for carbondata-hadoop module
This is an automated email from the ASF dual-hosted git repository.
manhua pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/carbondata.git
The following commit(s) were added to refs/heads/master by this push:
new 1cec1ee [CARBONDATA-3889] Cleanup code for carbondata-hadoop module
1cec1ee is described below
commit 1cec1ee7268cc18a98d8894119b38dc8df60644a
Author: QiangCai
AuthorDate: Tue Jul 7 15:48:10 2020 +0800
[CARBONDATA-3889] Cleanup code for carbondata-hadoop module
Why is this PR needed?
need cleanup code for carbondata-hadoop module
What changes were proposed in this PR?
Cleanup code for carbondata-hadoop module
Does this PR introduce any user interface change?
No
Is any new testcase added?
No
This closes #3827
---
.../carbondata/hadoop/AbstractRecordReader.java| 4 +-
.../carbondata/hadoop/CarbonMultiBlockSplit.java | 4 +-
.../apache/carbondata/hadoop/CarbonProjection.java | 7 +-
.../carbondata/hadoop/CarbonRecordReader.java | 6 +-
.../hadoop/api/CarbonFileInputFormat.java | 58 +++-
.../carbondata/hadoop/api/CarbonInputFormat.java | 71 --
.../hadoop/api/CarbonOutputCommitter.java | 33 ++---
.../hadoop/api/CarbonTableInputFormat.java | 25 +---
.../hadoop/api/CarbonTableOutputFormat.java| 65 -
.../hadoop/stream/CarbonStreamInputFormat.java | 4 +-
.../hadoop/stream/CarbonStreamUtils.java | 4 +-
.../hadoop/stream/StreamBlockletReader.java| 6 +-
.../hadoop/stream/StreamRecordReader.java | 26 ++--
.../carbondata/hadoop/testutil/StoreCreator.java | 150 +
.../hadoop/util/CarbonInputFormatUtil.java | 31 +
.../hadoop/util/CarbonInputSplitTaskInfo.java | 40 +++---
.../hadoop/util/CarbonVectorizedRecordReader.java | 12 +-
.../carbondata/spark/rdd/CarbonScanRDD.scala | 2 +-
.../datasources/SparkCarbonTableFormat.scala | 2 +-
19 files changed, 176 insertions(+), 374 deletions(-)
diff --git
a/hadoop/src/main/java/org/apache/carbondata/hadoop/AbstractRecordReader.java
b/hadoop/src/main/java/org/apache/carbondata/hadoop/AbstractRecordReader.java
index 5923ab7..15898d3 100644
---
a/hadoop/src/main/java/org/apache/carbondata/hadoop/AbstractRecordReader.java
+++
b/hadoop/src/main/java/org/apache/carbondata/hadoop/AbstractRecordReader.java
@@ -31,9 +31,7 @@ public abstract class AbstractRecordReader extends
RecordReader {
protected int rowCount = 0;
/**
- * This method will log query result count and querytime
- * @param recordCount
- * @param recorder
+ * This method will log query result count and query time
*/
public void logStatistics(int recordCount, QueryStatisticsRecorder recorder)
{
// result size
diff --git
a/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonMultiBlockSplit.java
b/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonMultiBlockSplit.java
index 64901cf..f6a11a2 100644
---
a/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonMultiBlockSplit.java
+++
b/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonMultiBlockSplit.java
@@ -139,8 +139,8 @@ public class CarbonMultiBlockSplit extends InputSplit
implements Serializable, W
}
getLocationIfNull();
out.writeInt(locations.length);
-for (int i = 0; i < locations.length; i++) {
- out.writeUTF(locations[i]);
+for (String location : locations) {
+ out.writeUTF(location);
}
out.writeInt(fileFormat.ordinal());
}
diff --git
a/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonProjection.java
b/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonProjection.java
index 13ccaa9..12c03c0 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonProjection.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonProjection.java
@@ -18,6 +18,7 @@
package org.apache.carbondata.hadoop;
import java.io.Serializable;
+import java.util.Arrays;
import java.util.LinkedHashSet;
import java.util.Objects;
import java.util.Set;
@@ -29,16 +30,14 @@ public class CarbonProjection implements Serializable {
private static final long serialVersionUID = -4328676723039530713L;
- private Set columns = new LinkedHashSet<>();
+ private final Set columns = new LinkedHashSet<>();
public CarbonProjection() {
}
public CarbonProjection(String[] columnNames) {
Objects.requireNonNull(columnNames);
-for (String columnName : columnNames) {
- columns.add(columnName);
-}
+columns.addAll(Arrays.asList(columnNames));
}
public void addColumn(String column) {
diff --git
a/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonRecordReader.java
b/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonRecordReader.java
index 943bd76..2c5c821 100644
--- a/hadoop/src/main/java/org/apache/ca
[carbondata] branch master updated: [CARBONDATA-3889] Cleanup code for carbondata-streaming module
This is an automated email from the ASF dual-hosted git repository.
manhua pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/carbondata.git
The following commit(s) were added to refs/heads/master by this push:
new 23e2760 [CARBONDATA-3889] Cleanup code for carbondata-streaming module
23e2760 is described below
commit 23e2760981e7674c88eae8319ec102d5b9adb544
Author: QiangCai
AuthorDate: Sat Jul 4 23:37:21 2020 +0800
[CARBONDATA-3889] Cleanup code for carbondata-streaming module
Why is this PR needed?
need cleanup code in carbondata-streaming module
What changes were proposed in this PR?
Cleanup code in carbondata-streaming module
Does this PR introduce any user interface change?
No
Yes. (please explain the change and update document)
Is any new testcase added?
No
Yes
This closes #3826
---
streaming/pom.xml | 19 +-
.../streaming/CarbonStreamRecordWriter.java| 6 +-
.../carbondata/streaming/StreamBlockletWriter.java | 22 +--
.../streaming/segment/StreamSegment.java | 210 +
.../streaming/parser/FieldConverter.scala | 2 +-
.../streaming/parser/RowStreamParserImp.scala | 16 +-
.../streaming/CarbonStreamOutputFormatTest.java| 12 +-
7 files changed, 132 insertions(+), 155 deletions(-)
diff --git a/streaming/pom.xml b/streaming/pom.xml
index 92cbed4..84fffea 100644
--- a/streaming/pom.xml
+++ b/streaming/pom.xml
@@ -1,3 +1,20 @@
+
+
http://maven.apache.org/POM/4.0.0;
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance;
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd;>
carbondata-parent
@@ -95,7 +112,7 @@
org.apache.maven.plugins
maven-surefire-plugin
2.18
-
+
${project.build.directory}/surefire-reports
-Xmx3g -XX:MaxPermSize=512m
-XX:ReservedCodeCacheSize=512m
diff --git
a/streaming/src/main/java/org/apache/carbondata/streaming/CarbonStreamRecordWriter.java
b/streaming/src/main/java/org/apache/carbondata/streaming/CarbonStreamRecordWriter.java
index 3209e8d..8db1bd7 100644
---
a/streaming/src/main/java/org/apache/carbondata/streaming/CarbonStreamRecordWriter.java
+++
b/streaming/src/main/java/org/apache/carbondata/streaming/CarbonStreamRecordWriter.java
@@ -82,7 +82,7 @@ public class CarbonStreamRecordWriter extends
RecordWriter {
private RowParser rowParser;
private BadRecordsLogger badRecordLogger;
private RowConverter converter;
- private CarbonRow currentRow = new CarbonRow(null);
+ private final CarbonRow currentRow = new CarbonRow(null);
// encoder
private DataField[] dataFields;
@@ -174,7 +174,7 @@ public class CarbonStreamRecordWriter extends
RecordWriter {
if (carbonFile.exists()) {
// if the file is existed, use the append api
outputStream = FileFactory.getDataOutputStreamUsingAppend(filePath);
- // get the compressor from the fileheader. In legacy store,
+ // get the compressor from the file header. In legacy store,
// the compressor name is not set and it use snappy compressor
FileHeader header = new CarbonHeaderReader(filePath).readHeader();
if (header.isSetCompressor_name()) {
@@ -329,7 +329,7 @@ public class CarbonStreamRecordWriter extends
RecordWriter {
if (output.getRowIndex() == -1) {
return;
}
-output.apppendBlocklet(outputStream);
+output.appendBlocklet(outputStream);
outputStream.flush();
if (!isClosed) {
batchMinMaxIndex = StreamSegment.mergeBlockletMinMax(
diff --git
a/streaming/src/main/java/org/apache/carbondata/streaming/StreamBlockletWriter.java
b/streaming/src/main/java/org/apache/carbondata/streaming/StreamBlockletWriter.java
index 89bf7c5..0391525 100644
---
a/streaming/src/main/java/org/apache/carbondata/streaming/StreamBlockletWriter.java
+++
b/streaming/src/main/java/org/apache/carbondata/streaming/StreamBlockletWriter.java
@@ -44,14 +44,14 @@ public class StreamBlockletWriter {
private byte[] buffer;
private int maxSize;
private int maxRowNum;
- private int rowSize;
+ private final int rowSize;
private int count = 0;
private int rowIndex = -1;
- private Compressor compressor;
+ private final Compressor compressor;
- private int dimCountWithoutComplex;
- private int measureCount;
- private DataType[] measureDataTypes;
+ private final int dimCountWithoutComplex;
+ private final int measureCount;
+ private final DataType[] measureDataTypes;
// blocklet level stats
ColumnPageStatsCollector[] dimStatsCollectors;
@@ -93,11 +93,11 @@ public class StreamBlockletWriter {
}
private void ensureCapacity(int space) {
-int newcount = space + count;
-if (newcount > buffer.length) {
- byte[] newbuf = new byte[Math.max(newcount, buffer.
[carbondata] branch master updated: [CARBONDATA-3889] Cleanup code for carbondata-common module
This is an automated email from the ASF dual-hosted git repository.
manhua pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/carbondata.git
The following commit(s) were added to refs/heads/master by this push:
new 6def83f [CARBONDATA-3889] Cleanup code for carbondata-common module
6def83f is described below
commit 6def83f85bef5890230376b75b02ef7d4d102aa0
Author: QiangCai
AuthorDate: Sat Jul 4 21:55:18 2020 +0800
[CARBONDATA-3889] Cleanup code for carbondata-common module
Why is this PR needed?
1.Field can be 'final'
2.Redundant 'if' statement
3.Empty method
4.Redundant 'throws' clause
5.Anonymous type can be replaced with lambda
6.Typo
7.Redundant String operation
What changes were proposed in this PR?
1.change field to be 'final'
2.change redundant 'if' statement
3.remove Empty method
4.remove redundant 'throws' clause
5.change anonymous type to lambda
6.fix typo
7.remove redundant String operation
Does this PR introduce any user interface change?
No
Is any new testcase added?
No
This closes #3825
---
.../main/java/org/apache/carbondata/common/Maps.java | 2 +-
.../carbondata/common/constants/LoggerAction.java | 2 +-
.../sql/MalformedCarbonCommandException.java | 2 +-
.../apache/carbondata/common/logging/LogService.java | 13 +++--
.../logging/impl/ExtendedRollingFileAppender.java | 19 +--
.../org/apache/carbondata/common/StringsSuite.java| 2 +-
.../common/logging/LogServiceFactoryTest_UT.java | 10 +-
.../impl/AuditExtendedRollingFileAppenderTest_UT.java | 7 +--
.../common/logging/impl/AuditLevelTest_UT.java| 8
.../impl/ExtendedRollingFileAppenderTest_UT.java | 6 +-
10 files changed, 15 insertions(+), 56 deletions(-)
diff --git a/common/src/main/java/org/apache/carbondata/common/Maps.java
b/common/src/main/java/org/apache/carbondata/common/Maps.java
index 4e76192..2fefbad 100644
--- a/common/src/main/java/org/apache/carbondata/common/Maps.java
+++ b/common/src/main/java/org/apache/carbondata/common/Maps.java
@@ -25,7 +25,7 @@ import
org.apache.carbondata.common.annotations.InterfaceAudience;
public class Maps {
/**
- * Return value if key is contained in the map, else return defauleValue.
+ * Return value if key is contained in the map, else return defaultValue.
* This is added to avoid JDK 8 dependency
*/
public static V getOrDefault(Map map, K key, V defaultValue) {
diff --git
a/common/src/main/java/org/apache/carbondata/common/constants/LoggerAction.java
b/common/src/main/java/org/apache/carbondata/common/constants/LoggerAction.java
index ca2e2e7..38a8143 100644
---
a/common/src/main/java/org/apache/carbondata/common/constants/LoggerAction.java
+++
b/common/src/main/java/org/apache/carbondata/common/constants/LoggerAction.java
@@ -26,7 +26,7 @@ public enum LoggerAction {
REDIRECT("REDIRECT"), // no null conversion moved to bad record and written
to raw csv
IGNORE("IGNORE"), // no null conversion moved to bad record and not written
to raw csv
FAIL("FAIL"); //data loading will fail if a bad record is found
- private String name;
+ private final String name;
LoggerAction(String name) {
this.name = name;
diff --git
a/common/src/main/java/org/apache/carbondata/common/exceptions/sql/MalformedCarbonCommandException.java
b/common/src/main/java/org/apache/carbondata/common/exceptions/sql/MalformedCarbonCommandException.java
index 0394d29..6948d42 100644
---
a/common/src/main/java/org/apache/carbondata/common/exceptions/sql/MalformedCarbonCommandException.java
+++
b/common/src/main/java/org/apache/carbondata/common/exceptions/sql/MalformedCarbonCommandException.java
@@ -36,7 +36,7 @@ public class MalformedCarbonCommandException extends
Exception {
/**
* The Error message.
*/
- private String msg = "";
+ private final String msg;
/**
* Constructor
diff --git
a/common/src/main/java/org/apache/carbondata/common/logging/LogService.java
b/common/src/main/java/org/apache/carbondata/common/logging/LogService.java
index 55dade6..6102e94 100644
--- a/common/src/main/java/org/apache/carbondata/common/logging/LogService.java
+++ b/common/src/main/java/org/apache/carbondata/common/logging/LogService.java
@@ -35,7 +35,7 @@ public class LogService extends Logger {
private static String hostName;
private static String username;
- {
+ static {
try {
hostName = InetAddress.getLocalHost().getHostName();
} catch (UnknownHostException e) {
@@ -77,9 +77,9 @@ public class LogService extends Logger {
}
public void audit(String message) {
-String threadid = Thread.currentThread().getId() + "";
+String threadId = Thread.currentThread().getId() + "";
super.log(AuditLevel.AUDIT,
-
[carbondata] branch master updated: [CARBONDATA-3888] Move .flattened-pom.xml to target folder
This is an automated email from the ASF dual-hosted git repository.
manhua pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/carbondata.git
The following commit(s) were added to refs/heads/master by this push:
new 4e57b73 [CARBONDATA-3888] Move .flattened-pom.xml to target folder
4e57b73 is described below
commit 4e57b73cf6b739a314ea15012800018520e3448f
Author: QiangCai
AuthorDate: Sat Jul 4 21:13:24 2020 +0800
[CARBONDATA-3888] Move .flattened-pom.xml to target folder
Why is this PR needed?
after .flattened-pom.xml is generated in the project folder, it will impact
the project import of Intellij idea
What changes were proposed in this PR?
set outputDirectory of flatten-maven-plugin to project.build.directory
Does this PR introduce any user interface change?
No
Is any new testcase added?
No
This closes #3824
---
pom.xml | 1 +
1 file changed, 1 insertion(+)
diff --git a/pom.xml b/pom.xml
index 49ad5b5..7f85d0c 100644
--- a/pom.xml
+++ b/pom.xml
@@ -551,6 +551,7 @@
flatten-maven-plugin
1.2.2
+ ${project.build.directory}
