This is an automated email from the ASF dual-hosted git repository.
srowen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 1a3858a [SPARK-26985][CORE] Fix "access only some column of the all
of columns " for big endian architecture BUG
1a3858a is described below
commit 1a3858a7694a189f434846b132b828e902273620
Author: ketank-new <[email protected]>
AuthorDate: Tue Jun 25 08:24:10 2019 -0500
[SPARK-26985][CORE] Fix "access only some column of the all of columns "
for big endian architecture BUG
continuation to https://github.com/apache/spark/pull/24788
## What changes were proposed in this pull request?
Changes are related to BIG ENDIAN system
This changes are done to
identify s390x platform.
use byteorder to BIG_ENDIAN for big endian systems
changes for 2 are done in access functions putFloats() and putDouble()
## How was this patch tested?
Changes have been tested to build successfully on s390x as well x86
platform to make sure build is successful.
Closes #24861 from ketank-new/ketan_latest_v2.3.2.
Authored-by: ketank-new <[email protected]>
Signed-off-by: Sean Owen <[email protected]>
---
common/unsafe/src/main/java/org/apache/spark/unsafe/Platform.java | 2 +-
.../apache/spark/sql/execution/vectorized/OffHeapColumnVector.java | 4 ++--
.../org/apache/spark/sql/execution/vectorized/OnHeapColumnVector.java | 4 ++--
3 files changed, 5 insertions(+), 5 deletions(-)
diff --git a/common/unsafe/src/main/java/org/apache/spark/unsafe/Platform.java
b/common/unsafe/src/main/java/org/apache/spark/unsafe/Platform.java
index d98f08f..dc8d6e3 100644
--- a/common/unsafe/src/main/java/org/apache/spark/unsafe/Platform.java
+++ b/common/unsafe/src/main/java/org/apache/spark/unsafe/Platform.java
@@ -304,7 +304,7 @@ public final class Platform {
static {
boolean _unaligned;
String arch = System.getProperty("os.arch", "");
- if (arch.equals("ppc64le") || arch.equals("ppc64")) {
+ if (arch.equals("ppc64le") || arch.equals("ppc64") ||
arch.equals("s390x")) {
// Since java.nio.Bits.unaligned() doesn't return true on ppc (See
JDK-8165231), but
// ppc64 and ppc64le support it
_unaligned = true;
diff --git
a/sql/core/src/main/java/org/apache/spark/sql/execution/vectorized/OffHeapColumnVector.java
b/sql/core/src/main/java/org/apache/spark/sql/execution/vectorized/OffHeapColumnVector.java
index 5e0cf7d..3b919c7 100644
---
a/sql/core/src/main/java/org/apache/spark/sql/execution/vectorized/OffHeapColumnVector.java
+++
b/sql/core/src/main/java/org/apache/spark/sql/execution/vectorized/OffHeapColumnVector.java
@@ -417,7 +417,7 @@ public final class OffHeapColumnVector extends
WritableColumnVector {
Platform.copyMemory(src, Platform.BYTE_ARRAY_OFFSET + srcIndex,
null, data + rowId * 4L, count * 4L);
} else {
- ByteBuffer bb = ByteBuffer.wrap(src).order(ByteOrder.LITTLE_ENDIAN);
+ ByteBuffer bb = ByteBuffer.wrap(src).order(ByteOrder.BIG_ENDIAN);
long offset = data + 4L * rowId;
for (int i = 0; i < count; ++i, offset += 4) {
Platform.putFloat(null, offset, bb.getFloat(srcIndex + (4 * i)));
@@ -472,7 +472,7 @@ public final class OffHeapColumnVector extends
WritableColumnVector {
Platform.copyMemory(src, Platform.BYTE_ARRAY_OFFSET + srcIndex,
null, data + rowId * 8L, count * 8L);
} else {
- ByteBuffer bb = ByteBuffer.wrap(src).order(ByteOrder.LITTLE_ENDIAN);
+ ByteBuffer bb = ByteBuffer.wrap(src).order(ByteOrder.BIG_ENDIAN);
long offset = data + 8L * rowId;
for (int i = 0; i < count; ++i, offset += 8) {
Platform.putDouble(null, offset, bb.getDouble(srcIndex + (8 * i)));
diff --git
a/sql/core/src/main/java/org/apache/spark/sql/execution/vectorized/OnHeapColumnVector.java
b/sql/core/src/main/java/org/apache/spark/sql/execution/vectorized/OnHeapColumnVector.java
index 577eab6..1bf3126 100644
---
a/sql/core/src/main/java/org/apache/spark/sql/execution/vectorized/OnHeapColumnVector.java
+++
b/sql/core/src/main/java/org/apache/spark/sql/execution/vectorized/OnHeapColumnVector.java
@@ -396,7 +396,7 @@ public final class OnHeapColumnVector extends
WritableColumnVector {
Platform.copyMemory(src, Platform.BYTE_ARRAY_OFFSET + srcIndex,
floatData,
Platform.DOUBLE_ARRAY_OFFSET + rowId * 4L, count * 4L);
} else {
- ByteBuffer bb = ByteBuffer.wrap(src).order(ByteOrder.LITTLE_ENDIAN);
+ ByteBuffer bb = ByteBuffer.wrap(src).order(ByteOrder.BIG_ENDIAN);
for (int i = 0; i < count; ++i) {
floatData[i + rowId] = bb.getFloat(srcIndex + (4 * i));
}
@@ -445,7 +445,7 @@ public final class OnHeapColumnVector extends
WritableColumnVector {
Platform.copyMemory(src, Platform.BYTE_ARRAY_OFFSET + srcIndex,
doubleData,
Platform.DOUBLE_ARRAY_OFFSET + rowId * 8L, count * 8L);
} else {
- ByteBuffer bb = ByteBuffer.wrap(src).order(ByteOrder.LITTLE_ENDIAN);
+ ByteBuffer bb = ByteBuffer.wrap(src).order(ByteOrder.BIG_ENDIAN);
for (int i = 0; i < count; ++i) {
doubleData[i + rowId] = bb.getDouble(srcIndex + (8 * i));
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
