Repository: kylin Updated Branches: refs/heads/master 4adea1677 -> d2476f301
Update test VM settings Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d2476f30 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d2476f30 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d2476f30 Branch: refs/heads/master Commit: d2476f3010ea9ff6190c64c10f16c209b694c498 Parents: 4adea16 Author: Hongbin Ma <[email protected]> Authored: Thu Mar 24 16:51:31 2016 +0800 Committer: Hongbin Ma <[email protected]> Committed: Thu Mar 24 16:51:31 2016 +0800 ---------------------------------------------------------------------- .../DoggedCubeBuilderStressTest.java | 105 ------------------ kylin-it/pom.xml | 9 +- .../ITDoggedCubeBuilderStressTest.java | 107 +++++++++++++++++++ pom.xml | 2 +- 4 files changed, 116 insertions(+), 107 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/d2476f30/core-cube/src/test/java/org/apache/kylin/cube/inmemcubing/DoggedCubeBuilderStressTest.java ---------------------------------------------------------------------- diff --git a/core-cube/src/test/java/org/apache/kylin/cube/inmemcubing/DoggedCubeBuilderStressTest.java b/core-cube/src/test/java/org/apache/kylin/cube/inmemcubing/DoggedCubeBuilderStressTest.java deleted file mode 100644 index 8d8366e..0000000 --- a/core-cube/src/test/java/org/apache/kylin/cube/inmemcubing/DoggedCubeBuilderStressTest.java +++ /dev/null @@ -1,105 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.cube.inmemcubing; - -import java.io.IOException; -import java.util.List; -import java.util.Map; -import java.util.concurrent.ArrayBlockingQueue; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; - -import org.apache.kylin.common.KylinConfig; -import org.apache.kylin.common.util.LocalFileMetadataTestCase; -import org.apache.kylin.cube.CubeInstance; -import org.apache.kylin.cube.CubeManager; -import org.apache.kylin.dimension.Dictionary; -import org.apache.kylin.gridtable.GTRecord; -import org.apache.kylin.metadata.model.TblColRef; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - */ -public class DoggedCubeBuilderStressTest extends LocalFileMetadataTestCase { - - @SuppressWarnings("unused") - private static final Logger logger = LoggerFactory.getLogger(DoggedCubeBuilderStressTest.class); - - // CI sandbox memory is no more than 512MB, this many input should hit memory threshold - private static final int INPUT_ROWS = 200000; - private static final int THREADS = 4; - - private static CubeInstance cube; - private static String flatTable; - private static Map<TblColRef, Dictionary<String>> dictionaryMap; - - @BeforeClass - public static void before() throws IOException { - staticCreateTestMetadata(); - - KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv(); - CubeManager cubeManager = CubeManager.getInstance(kylinConfig); - - cube = cubeManager.getCube("test_kylin_cube_without_slr_left_join_empty"); - flatTable = "../examples/test_case_data/localmeta/data/flatten_data_for_without_slr_left_join.csv"; - dictionaryMap = InMemCubeBuilderTest.getDictionaryMap(cube, flatTable); - } - - @AfterClass - public static void after() throws Exception { - staticCleanupTestMetadata(); - } - - @Test - public void test() throws Exception { - - ArrayBlockingQueue<List<String>> queue = new ArrayBlockingQueue<List<String>>(1000); - ExecutorService executorService = Executors.newSingleThreadExecutor(); - long randSeed = System.currentTimeMillis(); - - DoggedCubeBuilder doggedBuilder = new DoggedCubeBuilder(cube.getDescriptor(), dictionaryMap); - doggedBuilder.setConcurrentThreads(THREADS); - - { - Future<?> future = executorService.submit(doggedBuilder.buildAsRunnable(queue, new NoopWriter())); - InMemCubeBuilderTest.feedData(cube, flatTable, queue, INPUT_ROWS, randSeed); - future.get(); - } - } - - class NoopWriter implements ICuboidWriter { - @Override - public void write(long cuboidId, GTRecord record) throws IOException { - } - - @Override - public void flush() { - - } - - @Override - public void close() { - - } - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/kylin/blob/d2476f30/kylin-it/pom.xml ---------------------------------------------------------------------- diff --git a/kylin-it/pom.xml b/kylin-it/pom.xml index 979224d..fe426a9 100644 --- a/kylin-it/pom.xml +++ b/kylin-it/pom.xml @@ -112,6 +112,13 @@ </dependency> <dependency> <groupId>org.apache.kylin</groupId> + <artifactId>kylin-core-cube</artifactId> + <type>test-jar</type> + <scope>test</scope> + <version>${project.parent.version}</version> + </dependency> + <dependency> + <groupId>org.apache.kylin</groupId> <artifactId>kylin-core-storage</artifactId> <type>test-jar</type> <scope>test</scope> @@ -282,7 +289,7 @@ <value>true</value> </property> </systemProperties> - <argLine>-Xms1G -Xmx4G -XX:PermSize=128M -XX:MaxPermSize=512M</argLine> + <argLine>-Xms1G -Xmx2G -XX:PermSize=128M -XX:MaxPermSize=512M</argLine> </configuration> </plugin> <plugin> http://git-wip-us.apache.org/repos/asf/kylin/blob/d2476f30/kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITDoggedCubeBuilderStressTest.java ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITDoggedCubeBuilderStressTest.java b/kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITDoggedCubeBuilderStressTest.java new file mode 100644 index 0000000..4291d91 --- /dev/null +++ b/kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITDoggedCubeBuilderStressTest.java @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kylin.cube.inmemcubing; + +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; + +import org.apache.kylin.common.KylinConfig; +import org.apache.kylin.common.util.LocalFileMetadataTestCase; +import org.apache.kylin.cube.CubeInstance; +import org.apache.kylin.cube.CubeManager; +import org.apache.kylin.dimension.Dictionary; +import org.apache.kylin.gridtable.GTRecord; +import org.apache.kylin.metadata.model.TblColRef; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * very time consuming + */ +public class ITDoggedCubeBuilderStressTest extends LocalFileMetadataTestCase { + + @SuppressWarnings("unused") + private static final Logger logger = LoggerFactory.getLogger(ITDoggedCubeBuilderStressTest.class); + + // CI sandbox memory is no more than 512MB, this many input should hit memory threshold + private static final int INPUT_ROWS = 200000; + private static final int THREADS = 4; + + private static CubeInstance cube; + private static String flatTable; + private static Map<TblColRef, Dictionary<String>> dictionaryMap; + + @BeforeClass + public static void before() throws IOException { + staticCreateTestMetadata(); + + KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv(); + CubeManager cubeManager = CubeManager.getInstance(kylinConfig); + + cube = cubeManager.getCube("test_kylin_cube_without_slr_left_join_empty"); + flatTable = "../examples/test_case_data/localmeta/data/flatten_data_for_without_slr_left_join.csv"; + dictionaryMap = InMemCubeBuilderTest.getDictionaryMap(cube, flatTable); + } + + @AfterClass + public static void after() throws Exception { + staticCleanupTestMetadata(); + } + + @Test + public void test() throws Exception { + + ArrayBlockingQueue<List<String>> queue = new ArrayBlockingQueue<List<String>>(1000); + ExecutorService executorService = Executors.newSingleThreadExecutor(); + long randSeed = System.currentTimeMillis(); + + DoggedCubeBuilder doggedBuilder = new DoggedCubeBuilder(cube.getDescriptor(), dictionaryMap); + doggedBuilder.setConcurrentThreads(THREADS); + + { + Future<?> future = executorService.submit(doggedBuilder.buildAsRunnable(queue, new NoopWriter())); + InMemCubeBuilderTest.feedData(cube, flatTable, queue, INPUT_ROWS, randSeed); + future.get(); + } + } + + class NoopWriter implements ICuboidWriter { + @Override + public void write(long cuboidId, GTRecord record) throws IOException { + } + + @Override + public void flush() { + + } + + @Override + public void close() { + + } + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/kylin/blob/d2476f30/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 7e521e1..593bbff 100644 --- a/pom.xml +++ b/pom.xml @@ -670,7 +670,7 @@ <value>false</value> </property> </systemProperties> - <argLine>-Xmx4G -XX:MaxPermSize=512M</argLine> + <argLine>-Xms1G -Xmx2G -XX:MaxPermSize=512M</argLine> </configuration> </plugin>
