[kylin] branch 2.5.x updated: KYLIN-3644 Fix SparkFactDistinct step NumberFormatException

[shaofengshi]

This is an automated email from the ASF dual-hosted git repository.

shaofengshi pushed a commit to branch 2.5.x
in repository https://gitbox.apache.org/repos/asf/kylin.git


The following commit(s) were added to refs/heads/2.5.x by this push:
     new 0411360  KYLIN-3644 Fix SparkFactDistinct step NumberFormatException
0411360 is described below

commit 0411360557ec2355448fa67c3ae18a9473179afa
Author: chao long <wayn...@qq.com>
AuthorDate: Thu Nov 1 16:43:11 2018 +0800

    KYLIN-3644 Fix SparkFactDistinct step NumberFormatException
---
 .../src/main/java/org/apache/kylin/cube/util/KeyValueBuilder.java    | 2 +-
 .../main/java/org/apache/kylin/engine/spark/SparkFactDistinct.java   | 5 ++++-
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git 
a/core-cube/src/main/java/org/apache/kylin/cube/util/KeyValueBuilder.java 
b/core-cube/src/main/java/org/apache/kylin/cube/util/KeyValueBuilder.java
index 0ba4fd8..0636a5c 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/util/KeyValueBuilder.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/util/KeyValueBuilder.java
@@ -56,7 +56,7 @@ public class KeyValueBuilder implements Serializable {
         }
     }
 
-    protected boolean isNull(String v) {
+    public boolean isNull(String v) {
         return nullStrs.contains(v);
     }
 
diff --git 
a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkFactDistinct.java
 
b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkFactDistinct.java
index 213cdfd..7f05234 100644
--- 
a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkFactDistinct.java
+++ 
b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkFactDistinct.java
@@ -58,6 +58,7 @@ import org.apache.kylin.cube.DimensionRangeInfo;
 import org.apache.kylin.cube.cuboid.CuboidUtil;
 import org.apache.kylin.cube.model.CubeDesc;
 import org.apache.kylin.cube.model.CubeJoinedFlatTableEnrich;
+import org.apache.kylin.cube.util.KeyValueBuilder;
 import org.apache.kylin.dict.DictionaryGenerator;
 import org.apache.kylin.dict.IDictionaryBuilder;
 import org.apache.kylin.engine.EngineFactory;
@@ -238,6 +239,7 @@ public class SparkFactDistinct extends AbstractApplication 
implements Serializab
         private Map<Integer, DimensionRangeInfo> dimensionRangeInfoMap;
         private ByteBuffer tmpbuf;
         private LongAccumulator bytesWritten;
+        private KeyValueBuilder keyValueBuilder;
 
         public FlatOutputFucntion(String cubeName, String segmentId, String 
metaurl, SerializableConfiguration conf, int samplingPercent, LongAccumulator 
bytesWritten) {
             this.cubeName = cubeName;
@@ -259,6 +261,7 @@ public class SparkFactDistinct extends AbstractApplication 
implements Serializab
                 CubeJoinedFlatTableEnrich intermediateTableDesc = new 
CubeJoinedFlatTableEnrich(
                         EngineFactory.getJoinedFlatTableDesc(cubeSegment), 
cubeDesc);
 
+                keyValueBuilder = new KeyValueBuilder(intermediateTableDesc);
                 reducerMapping = new 
FactDistinctColumnsReducerMapping(cubeInstance);
                 tmpbuf = ByteBuffer.allocate(4096);
 
@@ -304,7 +307,7 @@ public class SparkFactDistinct extends AbstractApplication 
implements Serializab
 
                 for (int i = 0; i < allCols.size(); i++) {
                     String fieldValue = row[columnIndex[i]];
-                    if (fieldValue == null)
+                    if (fieldValue == null || 
keyValueBuilder.isNull(fieldValue))
                         continue;
 
                     final DataType type = allCols.get(i).getType();