This is an automated email from the ASF dual-hosted git repository.
asdf2014 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-druid.git
The following commit(s) were added to refs/heads/master by this push:
new d525e5b Fix travis timeout in BufferHashGrouperTest (#6713)
d525e5b is described below
commit d525e5b18e800d9fa59d8b80dc6487142f32ad75
Author: Jihoon Son <[email protected]>
AuthorDate: Thu Dec 6 20:05:27 2018 -0800
Fix travis timeout in BufferHashGrouperTest (#6713)
* Fix travis timeout in BufferHashGrouperTest
* adjust buffer size
* adjust bufferSize and loadFactor
* increase memory
* add debug code
* cat error
* after script
* print logs
* print per 2 min
* use direct mem
* clean up
---
processing/pom.xml | 2 +-
.../epinephelinae/BufferHashGrouperTest.java | 68 +++++++---------------
2 files changed, 21 insertions(+), 49 deletions(-)
diff --git a/processing/pom.xml b/processing/pom.xml
index 4963e32..35f6dd0 100644
--- a/processing/pom.xml
+++ b/processing/pom.xml
@@ -195,7 +195,7 @@
<!-- set default options -->
<argLine>
-Xmx512m
- -XX:MaxDirectMemorySize=1500m
+ -XX:MaxDirectMemorySize=2500m
-Duser.language=en
-Duser.GroupByQueryRunnerTest.javacountry=US
-Dfile.encoding=UTF-8
diff --git
a/processing/src/test/java/org/apache/druid/query/groupby/epinephelinae/BufferHashGrouperTest.java
b/processing/src/test/java/org/apache/druid/query/groupby/epinephelinae/BufferHashGrouperTest.java
index c28657a..64b270d 100644
---
a/processing/src/test/java/org/apache/druid/query/groupby/epinephelinae/BufferHashGrouperTest.java
+++
b/processing/src/test/java/org/apache/druid/query/groupby/epinephelinae/BufferHashGrouperTest.java
@@ -20,16 +20,13 @@
package org.apache.druid.query.groupby.epinephelinae;
import com.google.common.base.Suppliers;
-import com.google.common.base.Throwables;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Lists;
import com.google.common.collect.Ordering;
-import com.google.common.io.Files;
import com.google.common.primitives.Ints;
import org.apache.druid.common.config.NullHandling;
import org.apache.druid.data.input.MapBasedRow;
-import org.apache.druid.java.util.common.ByteBufferUtils;
import org.apache.druid.query.aggregation.AggregatorFactory;
import org.apache.druid.query.aggregation.CountAggregatorFactory;
import org.apache.druid.query.aggregation.LongSumAggregatorFactory;
@@ -39,12 +36,7 @@ import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.IOException;
import java.nio.ByteBuffer;
-import java.nio.MappedByteBuffer;
-import java.nio.channels.FileChannel;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.List;
@@ -112,7 +104,7 @@ public class BufferHashGrouperTest
public void testGrowing()
{
final TestColumnSelectorFactory columnSelectorFactory =
GrouperTestUtil.newColumnSelectorFactory();
- final Grouper<Integer> grouper = makeGrouper(columnSelectorFactory, 10000,
2);
+ final Grouper<Integer> grouper = makeGrouper(columnSelectorFactory, 10000,
2, 0.75f);
final int expectedMaxSize = NullHandling.replaceWithDefault() ? 219 : 210;
columnSelectorFactory.setRow(new MapBasedRow(0, ImmutableMap.of("value",
10L)));
@@ -137,38 +129,29 @@ public class BufferHashGrouperTest
}
@Test
- public void testGrowing2()
+ public void testGrowingOverflowingInteger()
{
- final TestColumnSelectorFactory columnSelectorFactory =
GrouperTestUtil.newColumnSelectorFactory();
- final Grouper<Integer> grouper = makeGrouper(columnSelectorFactory,
2_000_000_000, 2);
- final int expectedMaxSize = NullHandling.replaceWithDefault() ? 40988516 :
39141224;
-
- columnSelectorFactory.setRow(new MapBasedRow(0, ImmutableMap.of("value",
10L)));
- for (int i = 0; i < expectedMaxSize; i++) {
- Assert.assertTrue(String.valueOf(i), grouper.aggregate(i).isOk());
- }
- Assert.assertFalse(grouper.aggregate(expectedMaxSize).isOk());
- }
-
- @Test
- public void testGrowing3()
- {
- final TestColumnSelectorFactory columnSelectorFactory =
GrouperTestUtil.newColumnSelectorFactory();
- final Grouper<Integer> grouper = makeGrouper(columnSelectorFactory,
Integer.MAX_VALUE, 2);
- final int expectedMaxSize = NullHandling.replaceWithDefault() ? 44938972 :
42955456;
-
- columnSelectorFactory.setRow(new MapBasedRow(0, ImmutableMap.of("value",
10L)));
- for (int i = 0; i < expectedMaxSize; i++) {
- Assert.assertTrue(String.valueOf(i), grouper.aggregate(i).isOk());
+ // This test checks the bug reported in
https://github.com/apache/incubator-druid/pull/4333 only when
+ // NullHandling.replaceWithDefault() is true
+ if (NullHandling.replaceWithDefault()) {
+ final TestColumnSelectorFactory columnSelectorFactory =
GrouperTestUtil.newColumnSelectorFactory();
+ // the buffer size below is chosen to test integer overflow in
ByteBufferHashTable.adjustTableWhenFull().
+ final Grouper<Integer> grouper = makeGrouper(columnSelectorFactory,
1_900_000_000, 2, 0.3f);
+ final int expectedMaxSize = 15323979;
+
+ columnSelectorFactory.setRow(new MapBasedRow(0, ImmutableMap.of("value",
10L)));
+ for (int i = 0; i < expectedMaxSize; i++) {
+ Assert.assertTrue(String.valueOf(i), grouper.aggregate(i).isOk());
+ }
+ Assert.assertFalse(grouper.aggregate(expectedMaxSize).isOk());
}
- Assert.assertFalse(grouper.aggregate(expectedMaxSize).isOk());
}
@Test
public void testNoGrowing()
{
final TestColumnSelectorFactory columnSelectorFactory =
GrouperTestUtil.newColumnSelectorFactory();
- final Grouper<Integer> grouper = makeGrouper(columnSelectorFactory, 10000,
Integer.MAX_VALUE);
+ final Grouper<Integer> grouper = makeGrouper(columnSelectorFactory, 10000,
Integer.MAX_VALUE, 0.75f);
final int expectedMaxSize = NullHandling.replaceWithDefault() ? 267 : 258;
columnSelectorFactory.setRow(new MapBasedRow(0, ImmutableMap.of("value",
10L)));
@@ -195,22 +178,11 @@ public class BufferHashGrouperTest
private BufferHashGrouper<Integer> makeGrouper(
TestColumnSelectorFactory columnSelectorFactory,
int bufferSize,
- int initialBuckets
+ int initialBuckets,
+ float maxLoadFactor
)
{
- final MappedByteBuffer buffer;
-
- try {
- final File file = temporaryFolder.newFile();
- try (final FileChannel channel = new
FileOutputStream(file).getChannel()) {
- channel.truncate(bufferSize);
- }
- buffer = Files.map(file, FileChannel.MapMode.READ_WRITE, bufferSize);
- closerRule.closeLater(() -> ByteBufferUtils.unmap(buffer));
- }
- catch (IOException e) {
- throw Throwables.propagate(e);
- }
+ final ByteBuffer buffer = ByteBuffer.allocateDirect(bufferSize);
final BufferHashGrouper<Integer> grouper = new BufferHashGrouper<>(
Suppliers.ofInstance(buffer),
@@ -221,7 +193,7 @@ public class BufferHashGrouperTest
new CountAggregatorFactory("count")
},
Integer.MAX_VALUE,
- 0.75f,
+ maxLoadFactor,
initialBuckets,
true
);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
