Repository: carbondata Updated Branches: refs/heads/master 694ee774c -> a86c54b3e
[CARBONDATA-1907] Avoid unnecessary logging to improve query performance for no dictionary non string columns Changes done to return null in case of no dictionary column for non string data types when data is empty.This is done to avoid excessive logging which is impacting the query performance. This closes #1679 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/a86c54b3 Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/a86c54b3 Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/a86c54b3 Branch: refs/heads/master Commit: a86c54b3e83fef0b45ed70c50ee7483f5cc0894c Parents: 694ee77 Author: manishgupta88 <[email protected]> Authored: Tue Dec 19 17:27:30 2017 +0530 Committer: kumarvishal <[email protected]> Committed: Wed Dec 20 14:06:09 2017 +0530 ---------------------------------------------------------------------- .../executor/impl/AbstractQueryExecutor.java | 11 ++++----- .../carbondata/core/util/DataTypeUtil.java | 24 ++++++++++++++++++++ .../carbondata/core/util/DataTypeUtilTest.java | 18 +++++++++++++++ 3 files changed, 46 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/carbondata/blob/a86c54b3/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java b/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java index 88a8238..cb193e4 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java @@ -27,7 +27,6 @@ import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; import org.apache.carbondata.common.CarbonIterator; import org.apache.carbondata.common.logging.LogService; @@ -544,12 +543,10 @@ public abstract class AbstractQueryExecutor<E> implements QueryExecutor<E> { } UnsafeMemoryManager.INSTANCE.freeMemoryAll(ThreadLocalTaskInfo.getCarbonTaskInfo().getTaskId()); if (null != queryProperties.executorService) { - queryProperties.executorService.shutdown(); - try { - queryProperties.executorService.awaitTermination(1, TimeUnit.HOURS); - } catch (InterruptedException e) { - throw new QueryExecutionException(e); - } + // In case of limit query when number of limit records is already found so executors + // must stop all the running execution otherwise it will keep running and will hit + // the query performance. + queryProperties.executorService.shutdownNow(); } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/a86c54b3/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java b/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java index d8c13a3..0e819c4 100644 --- a/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java +++ b/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java @@ -353,12 +353,26 @@ public final class DataTypeUtil { } else if (actualDataType == DataTypes.STRING) { return getDataTypeConverter().convertFromByteToUTF8String(dataInBytes); } else if (actualDataType == DataTypes.SHORT) { + // for non string type no dictionary column empty byte array is empty value + // so no need to parse + if (isEmptyByteArray(dataInBytes)) { + return null; + } return ByteUtil.toShort(dataInBytes, 0, dataInBytes.length); } else if (actualDataType == DataTypes.INT) { + if (isEmptyByteArray(dataInBytes)) { + return null; + } return ByteUtil.toInt(dataInBytes, 0, dataInBytes.length); } else if (actualDataType == DataTypes.LONG) { + if (isEmptyByteArray(dataInBytes)) { + return null; + } return ByteUtil.toLong(dataInBytes, 0, dataInBytes.length); } else if (actualDataType == DataTypes.TIMESTAMP) { + if (isEmptyByteArray(dataInBytes)) { + return null; + } return ByteUtil.toLong(dataInBytes, 0, dataInBytes.length) * 1000L; } else { return ByteUtil.toString(dataInBytes, 0, dataInBytes.length); @@ -372,6 +386,16 @@ public final class DataTypeUtil { } } + /** + * Method to check if byte array is empty + * + * @param dataInBytes + * @return + */ + private static boolean isEmptyByteArray(byte[] dataInBytes) { + return dataInBytes.length == 0; + } + /** * Below method will be used to convert the data passed to its actual data http://git-wip-us.apache.org/repos/asf/carbondata/blob/a86c54b3/core/src/test/java/org/apache/carbondata/core/util/DataTypeUtilTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/carbondata/core/util/DataTypeUtilTest.java b/core/src/test/java/org/apache/carbondata/core/util/DataTypeUtilTest.java index c944d18..bb528b9 100644 --- a/core/src/test/java/org/apache/carbondata/core/util/DataTypeUtilTest.java +++ b/core/src/test/java/org/apache/carbondata/core/util/DataTypeUtilTest.java @@ -98,6 +98,24 @@ public class DataTypeUtilTest { } + @Test public void testGetDataBasedOnDataTypeForNoDictionaryColumn() { + Object result = DataTypeUtil.getDataBasedOnDataTypeForNoDictionaryColumn(new byte[0], + DataTypes.INT); + assert (result == null); + result = DataTypeUtil.getDataBasedOnDataTypeForNoDictionaryColumn(new byte[0], + DataTypes.SHORT); + assert (result == null); + result = DataTypeUtil.getDataBasedOnDataTypeForNoDictionaryColumn(new byte[0], + DataTypes.LONG); + assert (result == null); + result = DataTypeUtil.getDataBasedOnDataTypeForNoDictionaryColumn(new byte[0], + DataTypes.TIMESTAMP); + assert (result == null); + result = DataTypeUtil.getDataBasedOnDataTypeForNoDictionaryColumn(new byte[0], + DataTypes.STRING); + assert (result != null); + } + }
