This is an automated email from the ASF dual-hosted git repository.
cwylie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/master by this push:
new 2219e68fa3 add backwards compat mode for frontCoded
stringEncodingStrategy (#13988)
2219e68fa3 is described below
commit 2219e68fa3c602eaad8f5ba84acc36f8e2cabebc
Author: Clint Wylie <[email protected]>
AuthorDate: Tue Mar 28 14:44:44 2023 -0700
add backwards compat mode for frontCoded stringEncodingStrategy (#13988)
---
.../benchmark/FrontCodedIndexedBenchmark.java | 4 +-
docs/ingestion/ingestion-spec.md | 4 +-
.../segment/column/StringEncodingStrategies.java | 2 +-
.../segment/column/StringEncodingStrategy.java | 29 ++++++--
.../druid/segment/data/FrontCodedIndexed.java | 20 +++++-
.../segment/data/FrontCodedIndexedWriter.java | 33 ++++-----
.../segment/column/StringEncodingStrategyTest.java | 80 ++++++++++++++++++++++
.../druid/segment/data/FrontCodedIndexedTest.java | 49 +++++++------
8 files changed, 168 insertions(+), 53 deletions(-)
diff --git
a/benchmarks/src/test/java/org/apache/druid/benchmark/FrontCodedIndexedBenchmark.java
b/benchmarks/src/test/java/org/apache/druid/benchmark/FrontCodedIndexedBenchmark.java
index 00be920253..fe0e717ab1 100644
---
a/benchmarks/src/test/java/org/apache/druid/benchmark/FrontCodedIndexedBenchmark.java
+++
b/benchmarks/src/test/java/org/apache/druid/benchmark/FrontCodedIndexedBenchmark.java
@@ -139,7 +139,7 @@ public class FrontCodedIndexedBenchmark
new OnHeapMemorySegmentWriteOutMedium(),
ByteOrder.nativeOrder(),
"front-coded-4".equals(indexType) ? 4 : 16,
- false
+ FrontCodedIndexed.V0
);
frontCodedIndexedWriter.open();
@@ -147,7 +147,7 @@ public class FrontCodedIndexedBenchmark
new OnHeapMemorySegmentWriteOutMedium(),
ByteOrder.nativeOrder(),
"front-coded-incremental-buckets-4".equals(indexType) ? 4 : 16,
- true
+ FrontCodedIndexed.V1
);
frontCodedIndexedWriterIncrementalBuckets.open();
diff --git a/docs/ingestion/ingestion-spec.md b/docs/ingestion/ingestion-spec.md
index d0ae34eec3..22e4d167bb 100644
--- a/docs/ingestion/ingestion-spec.md
+++ b/docs/ingestion/ingestion-spec.md
@@ -477,7 +477,7 @@ The `indexSpec` object can include the following properties:
|-----|-----------|-------|
|bitmap|Compression format for bitmap indexes. Should be a JSON object with
`type` set to `roaring` or `concise`.|`{"type": "roaring"}`|
|dimensionCompression|Compression format for dimension columns. Options are
`lz4`, `lzf`, `zstd`, or `uncompressed`.|`lz4`|
-|stringDictionaryEncoding|Encoding format for STRING value dictionaries used
by STRING and COMPLEX<json> columns. <br /><br />Example to enable front
coding: `{"type":"frontCoded", "bucketSize": 4}`<br />`bucketSize` is the
number of values to place in a bucket to perform delta encoding. Must be a
power of 2, maximum is 128. Defaults to 4.<br /><br />See [Front
coding](#front-coding) for more information.|`{"type":"utf8"}`|
+|stringDictionaryEncoding|Encoding format for STRING value dictionaries used
by STRING and COMPLEX<json> columns. <br /><br />Example to enable front
coding: `{"type":"frontCoded", "bucketSize": 4}`<br />`bucketSize` is the
number of values to place in a bucket to perform delta encoding. Must be a
power of 2, maximum is 128. Defaults to 4.<br /> `formatVersion` can specify
older versions for backwards compatibility during rolling upgrades, valid
options are `0` and `1`. Defaults to [...]
|metricCompression|Compression format for primitive type metric columns.
Options are `lz4`, `lzf`, `zstd`, `uncompressed`, or `none` (which is more
efficient than `uncompressed`, but not supported by older versions of
Druid).|`lz4`|
|longEncoding|Encoding format for long-typed columns. Applies regardless of
whether they are dimensions or metrics. Options are `auto` or `longs`. `auto`
encodes the values using offset or lookup table depending on column
cardinality, and store them with variable size. `longs` stores the value as-is
with 8 bytes each.|`longs`|
|jsonCompression|Compression format to use for nested column raw data. Options
are `lz4`, `lzf`, `zstd`, or `uncompressed`.|`lz4`|
@@ -488,7 +488,7 @@ Front coding is an experimental feature starting in version
25.0. Front coding i
You can enable front coding with all types of ingestion. For information on
defining an `indexSpec` in a query context, see [SQL-based ingestion
reference](../multi-stage-query/reference.md#context-parameters).
-> Front coding is new to Druid 25.0 so the current recommendation is to enable
it in a staging environment and fully test your use case before using in
production. Segments created with front coding enabled are not compatible with
Druid versions older than 25.0.
+> Front coding was originally introduced in Druid 25.0, and an improved
'version 1' was introduced in Druid 26.0, with typically faster read speed and
smaller storage size. The current recommendation is to enable it in a staging
environment and fully test your use case before using in production. By
default, segments created with front coding enabled in Druid 26.0 are not
backwards compatible with Druid 25.0, and those created with Druid 25.0 are not
compatible with Druid versions older [...]
Beyond these properties, each ingestion method has its own specific tuning
properties. See the documentation for each
[ingestion method](./index.md#ingestion-methods) for details.
diff --git
a/processing/src/main/java/org/apache/druid/segment/column/StringEncodingStrategies.java
b/processing/src/main/java/org/apache/druid/segment/column/StringEncodingStrategies.java
index 0c04f0288c..c4a9b0661a 100644
---
a/processing/src/main/java/org/apache/druid/segment/column/StringEncodingStrategies.java
+++
b/processing/src/main/java/org/apache/druid/segment/column/StringEncodingStrategies.java
@@ -58,7 +58,7 @@ public class StringEncodingStrategies
writeoutMedium,
IndexIO.BYTE_ORDER,
strategy.getBucketSize(),
- true
+ strategy.getFormatVersion()
);
} else {
throw new ISE("Unknown encoding strategy: %s",
encodingStrategy.getType());
diff --git
a/processing/src/main/java/org/apache/druid/segment/column/StringEncodingStrategy.java
b/processing/src/main/java/org/apache/druid/segment/column/StringEncodingStrategy.java
index e850b0c362..a8246c9d75 100644
---
a/processing/src/main/java/org/apache/druid/segment/column/StringEncodingStrategy.java
+++
b/processing/src/main/java/org/apache/druid/segment/column/StringEncodingStrategy.java
@@ -24,6 +24,7 @@ import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.JsonSubTypes;
import com.fasterxml.jackson.annotation.JsonTypeInfo;
import org.apache.druid.java.util.common.ISE;
+import org.apache.druid.segment.data.FrontCodedIndexed;
import javax.annotation.Nullable;
import java.util.Objects;
@@ -84,20 +85,31 @@ public interface StringEncodingStrategy
class FrontCoded implements StringEncodingStrategy
{
- public static final int DEFAULT_BUCKET_SIZE = 4;
@JsonProperty
private final int bucketSize;
+ @JsonProperty
+ private final byte formatVersion;
+
@JsonCreator
public FrontCoded(
- @JsonProperty("bucketSize") @Nullable Integer bucketSize
+ @JsonProperty("bucketSize") @Nullable Integer bucketSize,
+ @JsonProperty("formatVersion") @Nullable Byte version
)
{
- this.bucketSize = bucketSize == null ? DEFAULT_BUCKET_SIZE : bucketSize;
+ this.bucketSize = bucketSize == null ?
FrontCodedIndexed.DEFAULT_BUCKET_SIZE : bucketSize;
if (Integer.bitCount(this.bucketSize) != 1) {
throw new ISE("bucketSize must be a power of two but was[%,d]",
bucketSize);
}
+ this.formatVersion = version == null
+ ? FrontCodedIndexed.DEFAULT_VERSION
+ : FrontCodedIndexed.validateVersion(version);
+ }
+
+ public FrontCoded(@Nullable Integer bucketSize)
+ {
+ this(bucketSize, null);
}
@JsonProperty
@@ -106,6 +118,12 @@ public interface StringEncodingStrategy
return bucketSize;
}
+ @JsonProperty
+ public byte getFormatVersion()
+ {
+ return formatVersion;
+ }
+
@Override
public String getType()
{
@@ -128,13 +146,13 @@ public interface StringEncodingStrategy
return false;
}
FrontCoded that = (FrontCoded) o;
- return bucketSize == that.bucketSize;
+ return bucketSize == that.bucketSize && formatVersion ==
that.formatVersion;
}
@Override
public int hashCode()
{
- return Objects.hash(bucketSize);
+ return Objects.hash(bucketSize, formatVersion);
}
@Override
@@ -142,6 +160,7 @@ public interface StringEncodingStrategy
{
return "FrontCoded{" +
"bucketSize=" + bucketSize +
+ ", formatVersion=" + formatVersion +
'}';
}
}
diff --git
a/processing/src/main/java/org/apache/druid/segment/data/FrontCodedIndexed.java
b/processing/src/main/java/org/apache/druid/segment/data/FrontCodedIndexed.java
index 27a84b5df3..bb135b0b9a 100644
---
a/processing/src/main/java/org/apache/druid/segment/data/FrontCodedIndexed.java
+++
b/processing/src/main/java/org/apache/druid/segment/data/FrontCodedIndexed.java
@@ -23,6 +23,7 @@ import com.google.common.base.Preconditions;
import com.google.common.base.Supplier;
import org.apache.druid.annotations.SuppressFBWarnings;
import org.apache.druid.common.config.NullHandling;
+import org.apache.druid.java.util.common.IAE;
import org.apache.druid.java.util.common.ISE;
import org.apache.druid.java.util.common.StringUtils;
import org.apache.druid.query.monomorphicprocessing.RuntimeShapeInspector;
@@ -76,11 +77,28 @@ import java.util.NoSuchElementException;
*/
public final class FrontCodedIndexed implements Indexed<ByteBuffer>
{
+ public static final byte V0 = 0;
+ public static final byte V1 = 1;
+ public static final byte DEFAULT_VERSION = V1;
+ public static final int DEFAULT_BUCKET_SIZE = 4;
+
+ public static byte validateVersion(byte version)
+ {
+ if (version != FrontCodedIndexed.V0 && version != FrontCodedIndexed.V1) {
+ throw new IAE(
+ "Unknown format version for FrontCodedIndexed [%s], must be [%s] or
[%s]",
+ version,
+ FrontCodedIndexed.V0,
+ FrontCodedIndexed.V1
+ );
+ }
+ return version;
+ }
public static Supplier<FrontCodedIndexed> read(ByteBuffer buffer, ByteOrder
ordering)
{
final ByteBuffer orderedBuffer = buffer.asReadOnlyBuffer().order(ordering);
final byte version = orderedBuffer.get();
- Preconditions.checkArgument(version == 0 || version == 1, "only V0 and V1
exist, encountered " + version);
+ Preconditions.checkArgument(version == V0 || version == V1, "only V0 and
V1 exist, encountered " + version);
final int bucketSize = Byte.toUnsignedInt(orderedBuffer.get());
final boolean hasNull = NullHandling.IS_NULL_BYTE == orderedBuffer.get();
final int numValues = VByte.readInt(orderedBuffer);
diff --git
a/processing/src/main/java/org/apache/druid/segment/data/FrontCodedIndexedWriter.java
b/processing/src/main/java/org/apache/druid/segment/data/FrontCodedIndexedWriter.java
index c5a26f3a59..8f711df610 100644
---
a/processing/src/main/java/org/apache/druid/segment/data/FrontCodedIndexedWriter.java
+++
b/processing/src/main/java/org/apache/druid/segment/data/FrontCodedIndexedWriter.java
@@ -59,7 +59,7 @@ public class FrontCodedIndexedWriter implements
DictionaryWriter<byte[]>
private final byte[][] bucketBuffer;
private final ByteBuffer getOffsetBuffer;
private final int div;
- private final boolean useIncrementalBuckets;
+ private final byte version;
@Nullable
private byte[] prevObject = null;
@@ -78,7 +78,7 @@ public class FrontCodedIndexedWriter implements
DictionaryWriter<byte[]>
SegmentWriteOutMedium segmentWriteOutMedium,
ByteOrder byteOrder,
int bucketSize,
- boolean useIncrementalBuckets
+ byte version
)
{
if (Integer.bitCount(bucketSize) != 1 || bucketSize < 1 || bucketSize >
128) {
@@ -91,7 +91,7 @@ public class FrontCodedIndexedWriter implements
DictionaryWriter<byte[]>
this.bucketBuffer = new byte[bucketSize][];
this.getOffsetBuffer = ByteBuffer.allocate(Integer.BYTES).order(byteOrder);
this.div = Integer.numberOfTrailingZeros(bucketSize);
- this.useIncrementalBuckets = useIncrementalBuckets;
+ this.version = FrontCodedIndexed.validateVersion(version);
}
@Override
@@ -124,9 +124,9 @@ public class FrontCodedIndexedWriter implements
DictionaryWriter<byte[]>
int written;
// write the bucket, growing scratch buffer as necessary
do {
- written = useIncrementalBuckets
- ? writeIncrementalBucket(scratch, bucketBuffer, bucketSize)
- : writeBucket(scratch, bucketBuffer, bucketSize);
+ written = version == FrontCodedIndexed.V1
+ ? writeBucketV1(scratch, bucketBuffer, bucketSize)
+ : writeBucketV0(scratch, bucketBuffer, bucketSize);
if (written < 0) {
growScratch();
}
@@ -170,14 +170,7 @@ public class FrontCodedIndexedWriter implements
DictionaryWriter<byte[]>
flush();
}
resetScratch();
-
- if (useIncrementalBuckets) {
- // version 1 is incremental buckets
- scratch.put((byte) 1);
- } else {
- // version 0 all values are prefixed on first bucket value
- scratch.put((byte) 0);
- }
+ scratch.put(version);
scratch.put((byte) bucketSize);
scratch.put(hasNulls ? NullHandling.IS_NULL_BYTE :
NullHandling.IS_NOT_NULL_BYTE);
VByte.writeInt(scratch, numWritten);
@@ -222,7 +215,7 @@ public class FrontCodedIndexedWriter implements
DictionaryWriter<byte[]>
final ByteBuffer bucketBuffer =
ByteBuffer.allocate(bucketBytesSize).order(byteOrder);
valuesOut.readFully(startOffset, bucketBuffer);
bucketBuffer.clear();
- final ByteBuffer valueBuffer = useIncrementalBuckets
+ final ByteBuffer valueBuffer = version == FrontCodedIndexed.V1
? getFromBucketV1(bucketBuffer,
relativeIndex, bucketSize)
:
FrontCodedIndexed.getFromBucketV0(bucketBuffer, relativeIndex);
final byte[] valueBytes = new byte[valueBuffer.limit() -
valueBuffer.position()];
@@ -248,9 +241,9 @@ public class FrontCodedIndexedWriter implements
DictionaryWriter<byte[]>
int written;
do {
int flushSize = remainder == 0 ? bucketSize : remainder;
- written = useIncrementalBuckets
- ? writeIncrementalBucket(scratch, bucketBuffer, flushSize)
- : writeBucket(scratch, bucketBuffer, flushSize);
+ written = version == FrontCodedIndexed.V1
+ ? writeBucketV1(scratch, bucketBuffer, flushSize)
+ : writeBucketV0(scratch, bucketBuffer, flushSize);
if (written < 0) {
growScratch();
}
@@ -283,7 +276,7 @@ public class FrontCodedIndexedWriter implements
DictionaryWriter<byte[]>
*
* Uses {@link VByte} encoded integers to indicate prefix length and value
length.
*/
- public static int writeBucket(ByteBuffer buffer, byte[][] values, int
numValues)
+ public static int writeBucketV0(ByteBuffer buffer, byte[][] values, int
numValues)
{
int written = 0;
byte[] first = null;
@@ -333,7 +326,7 @@ public class FrontCodedIndexedWriter implements
DictionaryWriter<byte[]>
*
* Uses {@link VByte} encoded integers to indicate prefix length and value
length.
*/
- public static int writeIncrementalBucket(ByteBuffer buffer, byte[][] values,
int numValues)
+ public static int writeBucketV1(ByteBuffer buffer, byte[][] values, int
numValues)
{
int written = 0;
byte[] prev = null;
diff --git
a/processing/src/test/java/org/apache/druid/segment/column/StringEncodingStrategyTest.java
b/processing/src/test/java/org/apache/druid/segment/column/StringEncodingStrategyTest.java
new file mode 100644
index 0000000000..e2b219c052
--- /dev/null
+++
b/processing/src/test/java/org/apache/druid/segment/column/StringEncodingStrategyTest.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.segment.column;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import nl.jqno.equalsverifier.EqualsVerifier;
+import org.apache.druid.jackson.DefaultObjectMapper;
+import org.apache.druid.segment.data.FrontCodedIndexed;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class StringEncodingStrategyTest
+{
+ private static final ObjectMapper JSON_MAPPER = new DefaultObjectMapper();
+
+
+ @Test
+ public void testUtf8Serde() throws JsonProcessingException
+ {
+ StringEncodingStrategy utf8 = new StringEncodingStrategy.Utf8();
+ String there = JSON_MAPPER.writeValueAsString(utf8);
+ StringEncodingStrategy andBackAgain = JSON_MAPPER.readValue(there,
StringEncodingStrategy.class);
+ Assert.assertEquals(utf8, andBackAgain);
+ }
+
+ @Test
+ public void testFrontCodedDefaultSerde() throws JsonProcessingException
+ {
+ StringEncodingStrategy frontCoded = new
StringEncodingStrategy.FrontCoded(null, null);
+ String there = JSON_MAPPER.writeValueAsString(frontCoded);
+ StringEncodingStrategy andBackAgain = JSON_MAPPER.readValue(there,
StringEncodingStrategy.class);
+ Assert.assertEquals(frontCoded, andBackAgain);
+ Assert.assertEquals(FrontCodedIndexed.DEFAULT_BUCKET_SIZE,
((StringEncodingStrategy.FrontCoded) andBackAgain).getBucketSize());
+ Assert.assertEquals(FrontCodedIndexed.DEFAULT_VERSION,
((StringEncodingStrategy.FrontCoded) andBackAgain).getFormatVersion());
+
+ // this next assert seems silly, but its a sanity check to make us think
hard before changing the default version,
+ // to make us think of the backwards compatibility implications, as new
versions of segment format stuff cannot be
+ // downgraded to older versions of Druid and still read
+ Assert.assertEquals(FrontCodedIndexed.V1,
FrontCodedIndexed.DEFAULT_VERSION);
+ }
+
+ @Test
+ public void testFrontCodedFormatSerde() throws JsonProcessingException
+ {
+ StringEncodingStrategy frontCodedV0 = new
StringEncodingStrategy.FrontCoded(16, FrontCodedIndexed.V0);
+ String there = JSON_MAPPER.writeValueAsString(frontCodedV0);
+ StringEncodingStrategy andBackAgain = JSON_MAPPER.readValue(there,
StringEncodingStrategy.class);
+ Assert.assertEquals(frontCodedV0, andBackAgain);
+
+ StringEncodingStrategy frontCodedV1 = new
StringEncodingStrategy.FrontCoded(8, FrontCodedIndexed.V1);
+ there = JSON_MAPPER.writeValueAsString(frontCodedV1);
+ andBackAgain = JSON_MAPPER.readValue(there, StringEncodingStrategy.class);
+ Assert.assertEquals(frontCodedV1, andBackAgain);
+ }
+
+ @Test
+ public void testEqualsAndHashcode()
+ {
+
EqualsVerifier.forClass(StringEncodingStrategy.Utf8.class).usingGetClass().verify();
+
EqualsVerifier.forClass(StringEncodingStrategy.FrontCoded.class).usingGetClass().verify();
+ }
+}
diff --git
a/processing/src/test/java/org/apache/druid/segment/data/FrontCodedIndexedTest.java
b/processing/src/test/java/org/apache/druid/segment/data/FrontCodedIndexedTest.java
index 4c1c51231a..98e6b4bbbf 100644
---
a/processing/src/test/java/org/apache/druid/segment/data/FrontCodedIndexedTest.java
+++
b/processing/src/test/java/org/apache/druid/segment/data/FrontCodedIndexedTest.java
@@ -47,20 +47,20 @@ public class FrontCodedIndexedTest extends
InitializedNullHandlingTest
public static Collection<Object[]> constructorFeeder()
{
return ImmutableList.of(
- new Object[]{ByteOrder.LITTLE_ENDIAN, true},
- new Object[]{ByteOrder.LITTLE_ENDIAN, false},
- new Object[]{ByteOrder.BIG_ENDIAN, true},
- new Object[]{ByteOrder.BIG_ENDIAN, false}
+ new Object[]{ByteOrder.LITTLE_ENDIAN, FrontCodedIndexed.V1},
+ new Object[]{ByteOrder.LITTLE_ENDIAN, FrontCodedIndexed.V0},
+ new Object[]{ByteOrder.BIG_ENDIAN, FrontCodedIndexed.V1},
+ new Object[]{ByteOrder.BIG_ENDIAN, FrontCodedIndexed.V0}
);
}
private final ByteOrder order;
- private final boolean useIncrementalBuckets;
+ private final byte version;
- public FrontCodedIndexedTest(ByteOrder byteOrder, boolean
useIncrementalBuckets)
+ public FrontCodedIndexedTest(ByteOrder byteOrder, byte version)
{
this.order = byteOrder;
- this.useIncrementalBuckets = useIncrementalBuckets;
+ this.version = version;
}
@Test
@@ -68,7 +68,7 @@ public class FrontCodedIndexedTest extends
InitializedNullHandlingTest
{
ByteBuffer buffer = ByteBuffer.allocate(1 << 12).order(order);
List<String> theList = ImmutableList.of("hello", "helloo", "hellooo",
"hellooz", "helloozy");
- persistToBuffer(buffer, theList, 4, useIncrementalBuckets);
+ persistToBuffer(buffer, theList, 4, version);
buffer.position(0);
FrontCodedIndexed codedUtf8Indexed = FrontCodedIndexed.read(
@@ -99,7 +99,7 @@ public class FrontCodedIndexedTest extends
InitializedNullHandlingTest
{
ByteBuffer buffer = ByteBuffer.allocate(1 << 12).order(order);
List<String> theList = ImmutableList.of("hello", "helloo", "hellooo",
"hellooz", "helloozy");
- persistToBuffer(buffer, theList, 16, useIncrementalBuckets);
+ persistToBuffer(buffer, theList, 16, version);
FrontCodedIndexed codedUtf8Indexed = FrontCodedIndexed.read(
buffer,
@@ -137,7 +137,7 @@ public class FrontCodedIndexedTest extends
InitializedNullHandlingTest
for (int i = 0; i < sizeBase + sizeAdjust; i++) {
values.add(IdUtils.getRandomId() + IdUtils.getRandomId() +
IdUtils.getRandomId() + IdUtils.getRandomId());
}
- persistToBuffer(buffer, values, bucketSize, useIncrementalBuckets);
+ persistToBuffer(buffer, values, bucketSize, version);
FrontCodedIndexed codedUtf8Indexed = FrontCodedIndexed.read(
buffer,
@@ -173,7 +173,7 @@ public class FrontCodedIndexedTest extends
InitializedNullHandlingTest
for (int i = 0; i < sizeBase + sizeAdjust; i++) {
values.add(IdUtils.getRandomId() + IdUtils.getRandomId() +
IdUtils.getRandomId() + IdUtils.getRandomId());
}
- persistToBuffer(buffer, values, bucketSize, useIncrementalBuckets);
+ persistToBuffer(buffer, values, bucketSize, version);
FrontCodedIndexed codedUtf8Indexed = FrontCodedIndexed.read(
buffer,
@@ -207,7 +207,7 @@ public class FrontCodedIndexedTest extends
InitializedNullHandlingTest
ByteBuffer buffer = ByteBuffer.allocate(1 << 12).order(order);
List<String> theList = ImmutableList.of("hello", "helloo", "hellooo",
"hellooz", "helloozy");
- persistToBuffer(buffer, theList, 4, useIncrementalBuckets);
+ persistToBuffer(buffer, theList, 4, version);
FrontCodedIndexed codedUtf8Indexed = FrontCodedIndexed.read(
buffer,
@@ -231,7 +231,7 @@ public class FrontCodedIndexedTest extends
InitializedNullHandlingTest
TreeSet<String> values = new TreeSet<>(GenericIndexed.STRING_STRATEGY);
values.add(null);
values.addAll(theList);
- persistToBuffer(buffer, values, 4, useIncrementalBuckets);
+ persistToBuffer(buffer, values, 4, version);
FrontCodedIndexed codedUtf8Indexed = FrontCodedIndexed.read(
buffer,
@@ -254,7 +254,7 @@ public class FrontCodedIndexedTest extends
InitializedNullHandlingTest
// "\uD83D\uDCA9" and "（請參見已被刪除版本）" are a regression test for
https://github.com/apache/druid/pull/13364
List<String> theList = ImmutableList.of("Győ-Moson-Sopron", "Győr",
"\uD83D\uDCA9", "（請參見已被刪除版本）");
- persistToBuffer(buffer, theList, 4, useIncrementalBuckets);
+ persistToBuffer(buffer, theList, 4, version);
buffer.position(0);
FrontCodedIndexed codedUtf8Indexed = FrontCodedIndexed.read(
@@ -282,7 +282,7 @@ public class FrontCodedIndexedTest extends
InitializedNullHandlingTest
{
ByteBuffer buffer = ByteBuffer.allocate(1 << 12).order(order);
List<String> theList = Collections.singletonList(null);
- persistToBuffer(buffer, theList, 4, useIncrementalBuckets);
+ persistToBuffer(buffer, theList, 4, version);
buffer.position(0);
FrontCodedIndexed codedUtf8Indexed = FrontCodedIndexed.read(
@@ -308,7 +308,7 @@ public class FrontCodedIndexedTest extends
InitializedNullHandlingTest
{
ByteBuffer buffer = ByteBuffer.allocate(1 << 6).order(order);
List<String> theList = Collections.emptyList();
- persistToBuffer(buffer, theList, 4, useIncrementalBuckets);
+ persistToBuffer(buffer, theList, 4, version);
buffer.position(0);
FrontCodedIndexed codedUtf8Indexed = FrontCodedIndexed.read(
@@ -351,7 +351,7 @@ public class FrontCodedIndexedTest extends
InitializedNullHandlingTest
values.add(IdUtils.getRandomId() + IdUtils.getRandomId() +
IdUtils.getRandomId() + IdUtils.getRandomId());
}
for (int bucketSize : bucketSizes) {
- persistToBuffer(buffer, values, bucketSize, useIncrementalBuckets);
+ persistToBuffer(buffer, values, bucketSize, version);
FrontCodedIndexed codedUtf8Indexed = FrontCodedIndexed.read(
buffer,
buffer.order()
@@ -389,7 +389,7 @@ public class FrontCodedIndexedTest extends
InitializedNullHandlingTest
medium,
ByteOrder.nativeOrder(),
0,
- useIncrementalBuckets
+ version
)
);
@@ -399,7 +399,7 @@ public class FrontCodedIndexedTest extends
InitializedNullHandlingTest
medium,
ByteOrder.nativeOrder(),
15,
- useIncrementalBuckets
+ version
)
);
@@ -409,12 +409,17 @@ public class FrontCodedIndexedTest extends
InitializedNullHandlingTest
medium,
ByteOrder.nativeOrder(),
256,
- useIncrementalBuckets
+ version
)
);
}
- private static long persistToBuffer(ByteBuffer buffer, Iterable<String>
sortedIterable, int bucketSize, boolean useIncrementalBuckets) throws
IOException
+ private static long persistToBuffer(
+ ByteBuffer buffer,
+ Iterable<String> sortedIterable,
+ int bucketSize,
+ byte version
+ ) throws IOException
{
Iterator<String> sortedStrings = sortedIterable.iterator();
buffer.position(0);
@@ -424,7 +429,7 @@ public class FrontCodedIndexedTest extends
InitializedNullHandlingTest
medium,
buffer.order(),
bucketSize,
- useIncrementalBuckets
+ version
);
writer.open();
int index = 0;
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
