This is an automated email from the ASF dual-hosted git repository.
kunalkapoor pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/carbondata.git
The following commit(s) were added to refs/heads/master by this push:
new 8a3c8a4 [CARBONDATA-3799] Fix inverted index cannot work with
adaptive encoding
8a3c8a4 is described below
commit 8a3c8a4be3ca0738014e4caf772515ed0c80b6fd
Author: ajantha-bhat <[email protected]>
AuthorDate: Wed May 6 13:33:21 2020 +0530
[CARBONDATA-3799] Fix inverted index cannot work with adaptive encoding
Why is this PR needed?
After PR #3638, Inverted index cannot work with adaptive encoding.
two issues are present
a) For Byte adaptive type (Not DirectByteBuffer), Encoded column page has
wrong result, as position() is used instead of limit()
b) For short adaptive type (DirectByteBuffer), result.array() will fail as
it is unsupported for direct byte buffer.
What changes were proposed in this PR?
For problem
a) use limit() instead of position()
b) write byte by byte instead of .array()
This closes #3746
---
.../core/datastore/page/encoding/adaptive/AdaptiveCodec.java | 9 ++++++++-
.../TestAdaptiveEncodingForPrimitiveTypes.scala | 12 ++++++++++++
2 files changed, 20 insertions(+), 1 deletion(-)
diff --git
a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveCodec.java
b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveCodec.java
index 274fb82..72c201e 100644
---
a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveCodec.java
+++
b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveCodec.java
@@ -162,7 +162,14 @@ public abstract class AdaptiveCodec implements
ColumnPageCodec {
ByteArrayOutputStream stream = new ByteArrayOutputStream();
DataOutputStream out = new DataOutputStream(stream);
if (null != indexStorage) {
- out.write(result.array(), 0, result.position());
+ if (result.isDirect()) {
+ // cannot apply result.array() on direct buffers
+ for (int i = result.position(); i < result.limit(); i++) {
+ out.writeByte(result.get(i));
+ }
+ } else {
+ out.write(result.array(), result.position(), result.limit());
+ }
if (indexStorage.getRowIdPageLengthInBytes() > 0) {
out.writeInt(indexStorage.getRowIdPageLengthInBytes());
short[] rowIdPage = (short[]) indexStorage.getRowIdPage();
diff --git
a/integration/spark/src/test/scala/org/apache/carbondata/integration/spark/testsuite/primitiveTypes/TestAdaptiveEncodingForPrimitiveTypes.scala
b/integration/spark/src/test/scala/org/apache/carbondata/integration/spark/testsuite/primitiveTypes/TestAdaptiveEncodingForPrimitiveTypes.scala
index dce3255..f38f193 100644
---
a/integration/spark/src/test/scala/org/apache/carbondata/integration/spark/testsuite/primitiveTypes/TestAdaptiveEncodingForPrimitiveTypes.scala
+++
b/integration/spark/src/test/scala/org/apache/carbondata/integration/spark/testsuite/primitiveTypes/TestAdaptiveEncodingForPrimitiveTypes.scala
@@ -392,6 +392,18 @@ class TestAdaptiveEncodingForPrimitiveTypes extends
QueryTest with BeforeAndAfte
sql("drop table if exists negativeTable")
}
+ test("test inverted index issue ") {
+ sql("DROP TABLE IF EXISTS source")
+ sql("CREATE TABLE source(intField int) stored as carbondata
TBLPROPERTIES('sort_columns'='intField','inverted_index'='intField')")
+ // this is stored as short data after adaptive encoding
+ sql("insert into source select 32000 union all select 0");
+ checkAnswer(sql("SELECT * from source"), Seq(Row(0), Row(32000)))
+ // this is stored as BYTE data after adaptive encoding
+ sql("insert into source select 32000 ");
+ checkAnswer(sql("SELECT * from source"), Seq(Row(0), Row(32000),
Row(32000)))
+ sql("DROP TABLE IF EXISTS source")
+ }
+
override def afterAll: Unit = {
CarbonProperties.getInstance()
.addProperty(CarbonCommonConstants.ENABLE_UNSAFE_SORT,
CarbonCommonConstants.ENABLE_UNSAFE_SORT_DEFAULT)
