spark git commit: [SPARK-23388][SQL] Support for Parquet Binary DecimalType in VectorizedColumnReader
Repository: spark
Updated Branches:
refs/heads/branch-2.3 70be6038d -> 4e138207e
[SPARK-23388][SQL] Support for Parquet Binary DecimalType in
VectorizedColumnReader
## What changes were proposed in this pull request?
Re-add support for parquet binary DecimalType in VectorizedColumnReader
## How was this patch tested?
Existing test suite
Author: James Thompson
Closes #20580 from jamesthomp/jt/add-back-binary-decimal.
(cherry picked from commit 5bb11411aec18b8d623e54caba5397d7cb8e89f0)
Signed-off-by: gatorsmile
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/4e138207
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/4e138207
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/4e138207
Branch: refs/heads/branch-2.3
Commit: 4e138207ebb11a08393c15e5e39f46a5dc1e7c66
Parents: 70be603
Author: James Thompson
Authored: Mon Feb 12 11:34:56 2018 -0800
Committer: gatorsmile
Committed: Mon Feb 12 11:35:06 2018 -0800
--
.../sql/execution/datasources/parquet/VectorizedColumnReader.java | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/4e138207/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedColumnReader.java
--
diff --git
a/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedColumnReader.java
b/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedColumnReader.java
index c120863..47dd625 100644
---
a/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedColumnReader.java
+++
b/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedColumnReader.java
@@ -444,7 +444,8 @@ public class VectorizedColumnReader {
// This is where we implement support for the valid type conversions.
// TODO: implement remaining type conversions
VectorizedValuesReader data = (VectorizedValuesReader) dataColumn;
-if (column.dataType() == DataTypes.StringType || column.dataType() ==
DataTypes.BinaryType) {
+if (column.dataType() == DataTypes.StringType || column.dataType() ==
DataTypes.BinaryType
+|| DecimalType.isByteArrayDecimalType(column.dataType())) {
defColumn.readBinarys(num, column, rowId, maxDefLevel, data);
} else if (column.dataType() == DataTypes.TimestampType) {
if (!shouldConvertTimestamps()) {
-
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
spark git commit: [SPARK-23388][SQL] Support for Parquet Binary DecimalType in VectorizedColumnReader
Repository: spark
Updated Branches:
refs/heads/master 4a4dd4f36 -> 5bb11411a
[SPARK-23388][SQL] Support for Parquet Binary DecimalType in
VectorizedColumnReader
## What changes were proposed in this pull request?
Re-add support for parquet binary DecimalType in VectorizedColumnReader
## How was this patch tested?
Existing test suite
Author: James Thompson
Closes #20580 from jamesthomp/jt/add-back-binary-decimal.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/5bb11411
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/5bb11411
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/5bb11411
Branch: refs/heads/master
Commit: 5bb11411aec18b8d623e54caba5397d7cb8e89f0
Parents: 4a4dd4f
Author: James Thompson
Authored: Mon Feb 12 11:34:56 2018 -0800
Committer: gatorsmile
Committed: Mon Feb 12 11:34:56 2018 -0800
--
.../sql/execution/datasources/parquet/VectorizedColumnReader.java | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/5bb11411/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedColumnReader.java
--
diff --git
a/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedColumnReader.java
b/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedColumnReader.java
index c120863..47dd625 100644
---
a/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedColumnReader.java
+++
b/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedColumnReader.java
@@ -444,7 +444,8 @@ public class VectorizedColumnReader {
// This is where we implement support for the valid type conversions.
// TODO: implement remaining type conversions
VectorizedValuesReader data = (VectorizedValuesReader) dataColumn;
-if (column.dataType() == DataTypes.StringType || column.dataType() ==
DataTypes.BinaryType) {
+if (column.dataType() == DataTypes.StringType || column.dataType() ==
DataTypes.BinaryType
+|| DecimalType.isByteArrayDecimalType(column.dataType())) {
defColumn.readBinarys(num, column, rowId, maxDefLevel, data);
} else if (column.dataType() == DataTypes.TimestampType) {
if (!shouldConvertTimestamps()) {
-
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
