http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/test/java/org/apache/kylin/job/hadoop/cube/MockupMapContext.java ---------------------------------------------------------------------- diff --git a/job/src/test/java/org/apache/kylin/job/hadoop/cube/MockupMapContext.java b/job/src/test/java/org/apache/kylin/job/hadoop/cube/MockupMapContext.java deleted file mode 100644 index 3f4c857..0000000 --- a/job/src/test/java/org/apache/kylin/job/hadoop/cube/MockupMapContext.java +++ /dev/null @@ -1,313 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.job.hadoop.cube; - -import java.io.IOException; -import java.net.URI; - -import org.apache.commons.lang.NotImplementedException; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.conf.Configuration.IntegerRanges; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.io.RawComparator; -import org.apache.hadoop.mapreduce.Counter; -import org.apache.hadoop.mapreduce.InputFormat; -import org.apache.hadoop.mapreduce.InputSplit; -import org.apache.hadoop.mapreduce.JobID; -import org.apache.hadoop.mapreduce.MapContext; -import org.apache.hadoop.mapreduce.Mapper; -import org.apache.hadoop.mapreduce.Mapper.Context; -import org.apache.hadoop.mapreduce.OutputCommitter; -import org.apache.hadoop.mapreduce.OutputFormat; -import org.apache.hadoop.mapreduce.Partitioner; -import org.apache.hadoop.mapreduce.Reducer; -import org.apache.hadoop.mapreduce.TaskAttemptID; -import org.apache.hadoop.mapreduce.lib.map.WrappedMapper; -import org.apache.hadoop.security.Credentials; -import org.apache.kylin.job.constant.BatchConstants; - -/** - * @author yangli9 - * - */ -@SuppressWarnings({ "rawtypes", "unchecked" }) -public class MockupMapContext { - - public static Context create(final Configuration hconf, String metadataUrl, String cubeName, final Object[] outKV) { - - hconf.set(BatchConstants.CFG_CUBE_NAME, cubeName); - - return new WrappedMapper().getMapContext(new MapContext() { - - @Override - public boolean nextKeyValue() throws IOException, InterruptedException { - throw new NotImplementedException(); - } - - @Override - public Object getCurrentKey() throws IOException, InterruptedException { - throw new NotImplementedException(); - } - - @Override - public Object getCurrentValue() throws IOException, InterruptedException { - throw new NotImplementedException(); - } - - @Override - public void write(Object key, Object value) throws IOException, InterruptedException { - System.out.println("Write -- k:" + key + ", v:" + value); - if (outKV != null) { - outKV[0] = key; - outKV[1] = value; - } - } - - @Override - public OutputCommitter getOutputCommitter() { - throw new NotImplementedException(); - } - - @Override - public TaskAttemptID getTaskAttemptID() { - throw new NotImplementedException(); - } - - @Override - public void setStatus(String msg) { - throw new NotImplementedException(); - } - - @Override - public String getStatus() { - throw new NotImplementedException(); - } - - @Override - public float getProgress() { - throw new NotImplementedException(); - } - - @Override - public Counter getCounter(Enum<?> counterName) { - throw new NotImplementedException(); - } - - @Override - public Counter getCounter(String groupName, String counterName) { - throw new NotImplementedException(); - } - - @Override - public Configuration getConfiguration() { - return hconf; - } - - @Override - public Credentials getCredentials() { - throw new NotImplementedException(); - } - - @Override - public JobID getJobID() { - throw new NotImplementedException(); - } - - @Override - public int getNumReduceTasks() { - throw new NotImplementedException(); - } - - @Override - public Path getWorkingDirectory() throws IOException { - throw new NotImplementedException(); - } - - @Override - public Class<?> getOutputKeyClass() { - throw new NotImplementedException(); - } - - @Override - public Class<?> getOutputValueClass() { - throw new NotImplementedException(); - } - - @Override - public Class<?> getMapOutputKeyClass() { - throw new NotImplementedException(); - } - - @Override - public Class<?> getMapOutputValueClass() { - throw new NotImplementedException(); - } - - @Override - public String getJobName() { - throw new NotImplementedException(); - } - - @Override - public Class<? extends InputFormat<?, ?>> getInputFormatClass() throws ClassNotFoundException { - throw new NotImplementedException(); - } - - @Override - public Class<? extends Mapper<?, ?, ?, ?>> getMapperClass() throws ClassNotFoundException { - throw new NotImplementedException(); - } - - @Override - public Class<? extends Reducer<?, ?, ?, ?>> getCombinerClass() throws ClassNotFoundException { - throw new NotImplementedException(); - } - - @Override - public Class<? extends Reducer<?, ?, ?, ?>> getReducerClass() throws ClassNotFoundException { - throw new NotImplementedException(); - } - - @Override - public Class<? extends OutputFormat<?, ?>> getOutputFormatClass() throws ClassNotFoundException { - throw new NotImplementedException(); - } - - @Override - public Class<? extends Partitioner<?, ?>> getPartitionerClass() throws ClassNotFoundException { - throw new NotImplementedException(); - } - - @Override - public RawComparator<?> getSortComparator() { - throw new NotImplementedException(); - } - - @Override - public String getJar() { - throw new NotImplementedException(); - } - - @Override - public RawComparator<?> getGroupingComparator() { - throw new NotImplementedException(); - } - - @Override - public boolean getJobSetupCleanupNeeded() { - throw new NotImplementedException(); - } - - @Override - public boolean getTaskCleanupNeeded() { - throw new NotImplementedException(); - } - - @Override - public boolean getProfileEnabled() { - throw new NotImplementedException(); - } - - @Override - public String getProfileParams() { - throw new NotImplementedException(); - } - - @Override - public IntegerRanges getProfileTaskRange(boolean isMap) { - throw new NotImplementedException(); - } - - @Override - public String getUser() { - throw new NotImplementedException(); - } - - @Override - public boolean getSymlink() { - throw new NotImplementedException(); - } - - @Override - public Path[] getArchiveClassPaths() { - throw new NotImplementedException(); - } - - @Override - public URI[] getCacheArchives() throws IOException { - throw new NotImplementedException(); - } - - @Override - public URI[] getCacheFiles() throws IOException { - throw new NotImplementedException(); - } - - @Override - public Path[] getLocalCacheArchives() throws IOException { - throw new NotImplementedException(); - } - - @Override - public Path[] getLocalCacheFiles() throws IOException { - throw new NotImplementedException(); - } - - @Override - public Path[] getFileClassPaths() { - throw new NotImplementedException(); - } - - @Override - public String[] getArchiveTimestamps() { - throw new NotImplementedException(); - } - - @Override - public String[] getFileTimestamps() { - throw new NotImplementedException(); - } - - @Override - public int getMaxMapAttempts() { - throw new NotImplementedException(); - } - - @Override - public int getMaxReduceAttempts() { - throw new NotImplementedException(); - } - - @Override - public void progress() { - throw new NotImplementedException(); - } - - @Override - public InputSplit getInputSplit() { - throw new NotImplementedException(); - } - - @Override - public RawComparator<?> getCombinerKeyGroupingComparator() { - throw new NotImplementedException(); - } - }); - } -}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/test/java/org/apache/kylin/job/hadoop/cube/NDCuboidJobTest.java ---------------------------------------------------------------------- diff --git a/job/src/test/java/org/apache/kylin/job/hadoop/cube/NDCuboidJobTest.java b/job/src/test/java/org/apache/kylin/job/hadoop/cube/NDCuboidJobTest.java deleted file mode 100644 index 8b5e78d..0000000 --- a/job/src/test/java/org/apache/kylin/job/hadoop/cube/NDCuboidJobTest.java +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.job.hadoop.cube; - -import static org.junit.Assert.*; - -import java.io.File; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileUtil; -import org.apache.hadoop.util.ToolRunner; -import org.apache.kylin.common.util.LocalFileMetadataTestCase; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -public class NDCuboidJobTest extends LocalFileMetadataTestCase { - - private Configuration conf; - - @Before - public void setup() throws Exception { - conf = new Configuration(); - BaseCuboidJobTest.setLocalMR(conf); - - createTestMetadata(); - } - - @After - public void after() throws Exception { - cleanupTestMetadata(); - } - - @Test - public void testJob8D() throws Exception { - String input = "src/test/resources/data/base_cuboid/"; - String output = "target/test-output/8d_cuboid"; - String cubeName = "test_kylin_cube_with_slr_1_new_segment"; - String segmentName = "20130331080000_20131212080000"; - String jobname = "8d_cuboid"; - String level = "1"; - - FileUtil.fullyDelete(new File(output)); - - String[] args = { "-input", input, "-cubename", cubeName, "-segmentname", segmentName, "-output", output, "-jobname", jobname, "-level", level }; - assertEquals("Job failed", 0, ToolRunner.run(conf, new NDCuboidJob(), args)); - } - - @Test - public void testJob7D() throws Exception { - final String input = "src/test/resources/data/8d_cuboid/"; - final String output = "target/test-output/7d_cuboid"; - final String cubeName = "test_kylin_cube_with_slr_1_new_segment"; - String segmentName = "20130331080000_20131212080000"; - String jobname = "7d_cuboid"; - String level = "2"; - - FileUtil.fullyDelete(new File(output)); - - String[] args = { "-input", input, "-cubename", cubeName, "-segmentname", segmentName, "-output", output, "-jobname", jobname, "-level", level }; - assertEquals("Job failed", 0, ToolRunner.run(conf, new NDCuboidJob(), args)); - } -} http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/test/java/org/apache/kylin/job/hadoop/cube/NDCuboidMapperTest.java ---------------------------------------------------------------------- diff --git a/job/src/test/java/org/apache/kylin/job/hadoop/cube/NDCuboidMapperTest.java b/job/src/test/java/org/apache/kylin/job/hadoop/cube/NDCuboidMapperTest.java deleted file mode 100644 index d60ec67..0000000 --- a/job/src/test/java/org/apache/kylin/job/hadoop/cube/NDCuboidMapperTest.java +++ /dev/null @@ -1,115 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.job.hadoop.cube; - -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -import java.io.File; -import java.io.IOException; -import java.util.Collection; -import java.util.List; - -import com.google.common.base.Function; -import com.google.common.collect.Collections2; -import org.apache.commons.io.FileUtils; -import org.apache.hadoop.io.Text; -import org.apache.hadoop.mrunit.mapreduce.MapReduceDriver; -import org.apache.hadoop.mrunit.types.Pair; -import org.apache.kylin.common.util.Bytes; -import org.apache.kylin.common.util.LocalFileMetadataTestCase; -import org.apache.kylin.job.constant.BatchConstants; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -import javax.annotation.Nullable; - -public class NDCuboidMapperTest extends LocalFileMetadataTestCase { - MapReduceDriver<Text, Text, Text, Text, Text, Text> mapReduceDriver; - - @Before - public void setUp() throws Exception { - createTestMetadata(); - - // hack for distributed cache - FileUtils.deleteDirectory(new File("../job/meta")); - FileUtils.copyDirectory(new File(getTestConfig().getMetadataUrl()), new File("../job/meta")); - - NDCuboidMapper mapper = new NDCuboidMapper(); - CuboidReducer reducer = new CuboidReducer(); - mapReduceDriver = MapReduceDriver.newMapReduceDriver(mapper, reducer); - } - - @After - public void after() throws Exception { - cleanupTestMetadata(); - FileUtils.deleteDirectory(new File("../job/meta")); - } - - @Test - public void testMapReduceWithSlr() throws IOException { - - String cubeName = "test_kylin_cube_with_slr_1_new_segment"; - String segmentName = "20130331080000_20131212080000"; - mapReduceDriver.getConfiguration().set(BatchConstants.CFG_CUBE_NAME, cubeName); - mapReduceDriver.getConfiguration().set(BatchConstants.CFG_CUBE_SEGMENT_NAME, segmentName); - - byte[] key = { 0, 0, 0, 0, 0, 0, 1, -1, 49, 48, 48, 48, 48, 48, 48, 48, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 11, 54, -105, 55, 13, 71, 114, 65, 66, 73, 78, 9, 9, 9, 9, 9, 9, 9, 9, 0, 10, 0 }; - byte[] value = { 14, 7, 23, -16, 56, 92, 114, -80, 118, 14, 7, 23, -16, 56, 92, 114, -80, 118, 14, 7, 23, -16, 56, 92, 114, -80, 118, 1, 1 }; - Pair<Text, Text> input1 = new Pair<Text, Text>(new Text(key), new Text(value)); - - mapReduceDriver.addInput(input1); - - List<Pair<Text, Text>> result = mapReduceDriver.run(); - - assertEquals(4, result.size()); - - byte[] resultKey = { 0, 0, 0, 0, 0, 0, 1, 127, 49, 48, 48, 48, 48, 48, 48, 48, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 55, 13, 71, 114, 65, 66, 73, 78, 9, 9, 9, 9, 9, 9, 9, 9, 0, 10, 0 }; - byte[] resultValue = { 14, 7, 23, -16, 56, 92, 114, -80, 118, 14, 7, 23, -16, 56, 92, 114, -80, 118, 14, 7, 23, -16, 56, 92, 114, -80, 118, 1, 1 }; - Pair<Text, Text> output1 = new Pair<Text, Text>(new Text(resultKey), new Text(resultValue)); - - - //As we will truncate decimal(KYLIN-766), value will no longer equals to resultValue - Collection<Text> keys = Collections2.transform(result, new Function<Pair<Text, Text>, Text>() { - @Nullable - @Override - public Text apply(Pair<Text, Text> input) { - return input.getFirst(); - } - }); - assertTrue(keys.contains(output1.getFirst())); - assertTrue(!result.contains(output1)); - - long[] keySet = new long[result.size()]; - - System.out.println(Bytes.toLong(new byte[] { 0, 0, 0, 0, 0, 0, 1, -1 })); - for (int i = 0; i < result.size(); i++) { - byte[] bytes = new byte[result.get(i).getFirst().getLength()]; - System.arraycopy(result.get(i).getFirst().getBytes(), 0, bytes, 0, result.get(i).getFirst().getLength()); - System.out.println(Bytes.toLong(bytes)); - keySet[i] = Bytes.toLong(bytes); - } - - // refer to CuboidSchedulerTest.testGetSpanningCuboid() - assertArrayEquals(new long[] { 383, 447, 503, 504 }, keySet); - - } -} http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/test/java/org/apache/kylin/job/hadoop/cube/RandomKeyDistributionMapperTest.java ---------------------------------------------------------------------- diff --git a/job/src/test/java/org/apache/kylin/job/hadoop/cube/RandomKeyDistributionMapperTest.java b/job/src/test/java/org/apache/kylin/job/hadoop/cube/RandomKeyDistributionMapperTest.java deleted file mode 100644 index 9a6fbbb..0000000 --- a/job/src/test/java/org/apache/kylin/job/hadoop/cube/RandomKeyDistributionMapperTest.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.job.hadoop.cube; - -import static org.junit.Assert.assertEquals; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; - -import org.apache.hadoop.io.LongWritable; -import org.apache.hadoop.io.Text; -import org.apache.hadoop.mrunit.mapreduce.MapDriver; -import org.apache.hadoop.mrunit.types.Pair; -import org.apache.kylin.job.constant.BatchConstants; -import org.apache.kylin.job.hadoop.invertedindex.RandomKeyDistributionMapper; -import org.junit.Before; -import org.junit.Test; - -/** - * @author ysong1 - * - */ -public class RandomKeyDistributionMapperTest { - - MapDriver<Text, Text, Text, LongWritable> mapDriver; - - @Before - @SuppressWarnings({ "rawtypes", "unchecked" }) - public void setUp() { - RandomKeyDistributionMapper mapper = new RandomKeyDistributionMapper(); - mapDriver = MapDriver.newMapDriver(mapper); - } - - @Test - public void test() throws IOException { - List<Text> data = new ArrayList<Text>(); - for (int i = 0; i < 1001; i++) { - data.add(new Text(String.valueOf(i))); - } - - for (Text t : data) { - mapDriver.addInput(t, new Text("abc")); - } - - mapDriver.getConfiguration().set(BatchConstants.MAPPER_SAMPLE_NUMBER, "100"); - List<Pair<Text, LongWritable>> result = mapDriver.run(); - assertEquals(100, result.size()); - - for (Pair<Text, LongWritable> p : result) { - System.out.println(p.getFirst()); - } - } - -} http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/test/java/org/apache/kylin/job/hadoop/cube/RandomKeyDistributionReducerTest.java ---------------------------------------------------------------------- diff --git a/job/src/test/java/org/apache/kylin/job/hadoop/cube/RandomKeyDistributionReducerTest.java b/job/src/test/java/org/apache/kylin/job/hadoop/cube/RandomKeyDistributionReducerTest.java deleted file mode 100644 index e5f0b7e..0000000 --- a/job/src/test/java/org/apache/kylin/job/hadoop/cube/RandomKeyDistributionReducerTest.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.job.hadoop.cube; - -import static org.junit.Assert.assertEquals; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; - -import org.apache.hadoop.io.LongWritable; -import org.apache.hadoop.io.Text; -import org.apache.hadoop.mrunit.mapreduce.ReduceDriver; -import org.apache.hadoop.mrunit.types.Pair; -import org.apache.kylin.job.constant.BatchConstants; -import org.apache.kylin.job.hadoop.invertedindex.RandomKeyDistributionReducer; -import org.junit.Before; -import org.junit.Test; - -/** - * @author ysong1 - * - */ -public class RandomKeyDistributionReducerTest { - ReduceDriver<Text, LongWritable, Text, LongWritable> reduceDriver; - - @Before - @SuppressWarnings({ "rawtypes", "unchecked" }) - public void setUp() { - RandomKeyDistributionReducer reducer = new RandomKeyDistributionReducer(); - reduceDriver = ReduceDriver.newReduceDriver(reducer); - } - - @Test - public void test() throws IOException { - List<Text> data = new ArrayList<Text>(); - for (int i = 0; i < 1001; i++) { - data.add(new Text(String.valueOf(i))); - } - for (Text t : data) { - reduceDriver.addInput(t, new ArrayList<LongWritable>()); - } - - reduceDriver.getConfiguration().set(BatchConstants.REGION_NUMBER, "2"); - List<Pair<Text, LongWritable>> result = reduceDriver.run(); - - assertEquals(2, result.size()); - - for (Pair<Text, LongWritable> p : result) { - System.out.println(p.getFirst()); - } - } - -} http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/test/java/org/apache/kylin/job/hadoop/cube/RangeKeyDistributionJobTest.java ---------------------------------------------------------------------- diff --git a/job/src/test/java/org/apache/kylin/job/hadoop/cube/RangeKeyDistributionJobTest.java b/job/src/test/java/org/apache/kylin/job/hadoop/cube/RangeKeyDistributionJobTest.java deleted file mode 100644 index 8e0c3bb..0000000 --- a/job/src/test/java/org/apache/kylin/job/hadoop/cube/RangeKeyDistributionJobTest.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.job.hadoop.cube; - -import static org.junit.Assert.assertEquals; - -import java.io.File; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileUtil; -import org.apache.hadoop.util.ToolRunner; -import org.apache.kylin.common.util.LocalFileMetadataTestCase; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -/** - * @author ysong1 - * - */ -public class RangeKeyDistributionJobTest extends LocalFileMetadataTestCase { - - private Configuration conf; - - @Before - public void setup() throws Exception { - conf = new Configuration(); - BaseCuboidJobTest.setLocalMR(conf); - - createTestMetadata(); - } - - @After - public void after() throws Exception { - cleanupTestMetadata(); - } - - @Test - public void testJob() throws Exception { - String input = "src/test/resources/data/base_cuboid/,src/test/resources/data/8d_cuboid/"; - String output = "target/test-output/key_distribution_range/"; - String jobname = "calculate_splits"; - String cubename = "test_kylin_cube_with_slr_ready"; - - FileUtil.fullyDelete(new File(output)); - - String[] args = { "-input", input, "-output", output, "-jobname", jobname, "-cubename", cubename }; - assertEquals("Job failed", 0, ToolRunner.run(conf, new RangeKeyDistributionJob(), args)); - } - -} http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/test/java/org/apache/kylin/job/hadoop/cube/RangeKeyDistributionMapperTest.java ---------------------------------------------------------------------- diff --git a/job/src/test/java/org/apache/kylin/job/hadoop/cube/RangeKeyDistributionMapperTest.java b/job/src/test/java/org/apache/kylin/job/hadoop/cube/RangeKeyDistributionMapperTest.java deleted file mode 100644 index 6cc8703..0000000 --- a/job/src/test/java/org/apache/kylin/job/hadoop/cube/RangeKeyDistributionMapperTest.java +++ /dev/null @@ -1,112 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.job.hadoop.cube; - -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertEquals; - -import java.io.File; -import java.io.IOException; -import java.util.List; - -import org.apache.hadoop.io.LongWritable; -import org.apache.hadoop.io.Text; -import org.apache.hadoop.mrunit.mapreduce.MapDriver; -import org.apache.hadoop.mrunit.types.Pair; -import org.junit.Before; -import org.junit.Test; - -/** - * @author ysong1 - * - */ -public class RangeKeyDistributionMapperTest { - - @SuppressWarnings("rawtypes") - MapDriver mapDriver; - String localTempDir = System.getProperty("java.io.tmpdir") + File.separator; - - @Before - public void setUp() { - RangeKeyDistributionMapper mapper = new RangeKeyDistributionMapper(); - mapDriver = MapDriver.newMapDriver(mapper); - } - - @SuppressWarnings("unchecked") - @Test - public void testMapperWithoutHeader() throws IOException { - - Text inputKey1 = new Text(new byte[] { 0, 0, 0, 0, 0, 0, 0, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 }); - Text inputKey2 = new Text(new byte[] { 0, 0, 0, 0, 0, 0, 0, 127, 11, 122, 1, 0, 22, 98, 1, 0, 121, 7 }); - Text inputKey3 = new Text(new byte[] { 2, 2, 2, 2, 2, 2, 2, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 }); - Text inputKey4 = new Text(new byte[] { 3, 3, 3, 3, 3, 3, 3, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 }); - Text inputKey5 = new Text(new byte[] { 4, 4, 4, 4, 4, 4, 4, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 }); - Text inputKey6 = new Text(new byte[] { 5, 5, 5, 5, 5, 5, 5, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 }); - Text inputKey7 = new Text(new byte[] { 6, 6, 6, 6, 6, 6, 6, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 }); - - mapDriver.addInput(inputKey1, new Text("abc")); - mapDriver.addInput(inputKey2, new Text("abc")); - mapDriver.addInput(inputKey3, new Text("abc")); - mapDriver.addInput(inputKey4, new Text("abc")); - mapDriver.addInput(inputKey5, new Text("abc")); - mapDriver.addInput(inputKey6, new Text("abc")); - mapDriver.addInput(inputKey7, new Text("abc")); - - List<Pair<Text, LongWritable>> result = mapDriver.run(); - - assertEquals(1, result.size()); - - byte[] key1 = result.get(0).getFirst().getBytes(); - LongWritable value1 = result.get(0).getSecond(); - assertArrayEquals(new byte[] { 6, 6, 6, 6, 6, 6, 6, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 }, key1); - assertEquals(147, value1.get()); - } - - @SuppressWarnings("unchecked") - @Test - public void testMapperWithHeader() throws IOException { - - Text inputKey1 = new Text(new byte[] { 0, 0, 0, 0, 0, 0, 0, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7, 0, 0, 0, 0, 0, 0, 0, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 }); - Text inputKey2 = new Text(new byte[] { 0, 0, 0, 0, 0, 0, 0, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7, 0, 0, 0, 0, 0, 0, 0, 127, 11, 122, 1, 0, 22, 98, 1, 0, 121, 7 }); - Text inputKey3 = new Text(new byte[] { 0, 0, 0, 0, 0, 0, 0, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7, 2, 2, 2, 2, 2, 2, 2, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 }); - Text inputKey4 = new Text(new byte[] { 0, 0, 0, 0, 0, 0, 0, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7, 3, 3, 3, 3, 3, 3, 3, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 }); - Text inputKey5 = new Text(new byte[] { 0, 0, 0, 0, 0, 0, 0, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7, 4, 4, 4, 4, 4, 4, 4, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 }); - Text inputKey6 = new Text(new byte[] { 0, 0, 0, 0, 0, 0, 0, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7, 5, 5, 5, 5, 5, 5, 5, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 }); - Text inputKey7 = new Text(new byte[] { 0, 0, 0, 0, 0, 0, 0, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7, 6, 6, 6, 6, 6, 6, 6, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 }); - - mapDriver.addInput(inputKey1, new Text("abc")); - mapDriver.addInput(inputKey2, new Text("abc")); - mapDriver.addInput(inputKey3, new Text("abc")); - mapDriver.addInput(inputKey4, new Text("abc")); - mapDriver.addInput(inputKey5, new Text("abc")); - mapDriver.addInput(inputKey6, new Text("abc")); - mapDriver.addInput(inputKey7, new Text("abc")); - - List<Pair<Text, LongWritable>> result = mapDriver.run(); - - assertEquals(1, result.size()); - - byte[] key1 = result.get(0).getFirst().getBytes(); - LongWritable value1 = result.get(0).getSecond(); - assertArrayEquals(new byte[] { 0, 0, 0, 0, 0, 0, 0, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7, 6, 6, 6, 6, 6, 6, 6, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 }, key1); - assertEquals(273, value1.get()); - - } - -} http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/test/java/org/apache/kylin/job/hadoop/cube/RangeKeyDistributionReducerTest.java ---------------------------------------------------------------------- diff --git a/job/src/test/java/org/apache/kylin/job/hadoop/cube/RangeKeyDistributionReducerTest.java b/job/src/test/java/org/apache/kylin/job/hadoop/cube/RangeKeyDistributionReducerTest.java deleted file mode 100644 index c1480a0..0000000 --- a/job/src/test/java/org/apache/kylin/job/hadoop/cube/RangeKeyDistributionReducerTest.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.job.hadoop.cube; - -import java.io.File; -import java.io.IOException; - -import org.apache.hadoop.io.LongWritable; -import org.apache.hadoop.io.Text; -import org.apache.hadoop.mrunit.mapreduce.ReduceDriver; -import org.junit.Before; -import org.junit.Test; - -/** - * @author ysong1 - * - */ -public class RangeKeyDistributionReducerTest { - - ReduceDriver<Text, LongWritable, Text, LongWritable> reduceDriver; - String localTempDir = System.getProperty("java.io.tmpdir") + File.separator; - - @Before - public void setUp() { - RangeKeyDistributionReducer reducer = new RangeKeyDistributionReducer(); - reduceDriver = ReduceDriver.newReduceDriver(reducer); - } - - @Test - public void testReducer() throws IOException { - // TODO - } -} http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/test/java/org/apache/kylin/job/hadoop/hbase/CreateHTableTest.java ---------------------------------------------------------------------- diff --git a/job/src/test/java/org/apache/kylin/job/hadoop/hbase/CreateHTableTest.java b/job/src/test/java/org/apache/kylin/job/hadoop/hbase/CreateHTableTest.java deleted file mode 100644 index 365a0d8..0000000 --- a/job/src/test/java/org/apache/kylin/job/hadoop/hbase/CreateHTableTest.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.job.hadoop.hbase; - -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertEquals; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.Path; -import org.apache.kylin.common.util.LocalFileMetadataTestCase; -import org.apache.kylin.job.hadoop.cube.BaseCuboidJobTest; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -/** - * @author George Song (ysong1) - * - */ -public class CreateHTableTest extends LocalFileMetadataTestCase { - - private Configuration conf; - - @Before - public void setup() throws Exception { - conf = new Configuration(); - BaseCuboidJobTest.setLocalMR(conf); - - this.createTestMetadata(); - - } - - @After - public void after() throws Exception { - this.cleanupTestMetadata(); - } - - @Test - public void testGetSplits() throws IllegalArgumentException, Exception { - CreateHTableJob c = new CreateHTableJob(); - - String input = "src/test/resources/partition_list/part-r-00000"; - - byte[][] splits = c.getSplits(conf, new Path(input)); - - assertEquals(497, splits.length); - assertArrayEquals(new byte[] { 0, 0, 0, 0, 0, 0, 15, -1, 11, 51, -45, 2 }, splits[0]); - assertArrayEquals(new byte[] { 0, 0, 0, 0, 0, 3, -1, -1, -54, -61, 109, -44, 1 }, splits[496]); - } - -} http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/test/java/org/apache/kylin/job/hadoop/hbase/TestHbaseClient.java ---------------------------------------------------------------------- diff --git a/job/src/test/java/org/apache/kylin/job/hadoop/hbase/TestHbaseClient.java b/job/src/test/java/org/apache/kylin/job/hadoop/hbase/TestHbaseClient.java deleted file mode 100644 index f2b9ed6..0000000 --- a/job/src/test/java/org/apache/kylin/job/hadoop/hbase/TestHbaseClient.java +++ /dev/null @@ -1,102 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.job.hadoop.hbase; - -import java.io.IOException; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HBaseConfiguration; -import org.apache.hadoop.hbase.client.HTable; -import org.apache.hadoop.hbase.client.Put; -import org.apache.kylin.common.util.Bytes; - -/** - * Created by hongbin on 5/15/14. - */ -public class TestHbaseClient { - - private static boolean reverse = false; - - public static void foo(int n, int k) { - int t = k; - if (n - k < k) { - t = n - k; - reverse = true; - } - boolean[] flags = new boolean[n]; - inner(flags, 0, t); - } - - private static void print(boolean[] flags) { - for (int i = 0; i < flags.length; i++) { - if (!reverse) { - if (flags[i]) - System.out.print("0"); - else - System.out.print("1"); - } else { - if (flags[i]) - System.out.print("1"); - else - System.out.print("0"); - - } - } - System.out.println(); - - } - - private static void inner(boolean[] flags, int start, int remaining) { - if (remaining <= 0) { - print(flags); - return; - } - - if (flags.length - start < remaining) { - return; - } - - // write at flags[start] - flags[start] = true; - inner(flags, start + 1, remaining - 1); - - // not write at flags[start] - flags[start] = false; - inner(flags, start + 1, remaining); - } - - public static void main(String[] args) throws IOException { - foo(6, 5); - foo(5, 2); - foo(3, 0); - - Configuration conf = HBaseConfiguration.create(); - conf.set("hbase.zookeeper.quorum", "hbase_host"); - conf.set("zookeeper.znode.parent", "/hbase-unsecure"); - - HTable table = new HTable(conf, "test1"); - Put put = new Put(Bytes.toBytes("row1")); - - put.add(Bytes.toBytes("colfam1"), Bytes.toBytes("qual1"), Bytes.toBytes("val1")); - put.add(Bytes.toBytes("colfam1"), Bytes.toBytes("qual2"), Bytes.toBytes("val2")); - - table.put(put); - table.close(); - } -} http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/test/java/org/apache/kylin/job/hadoop/hive/JoinedFlatTableTest.java ---------------------------------------------------------------------- diff --git a/job/src/test/java/org/apache/kylin/job/hadoop/hive/JoinedFlatTableTest.java b/job/src/test/java/org/apache/kylin/job/hadoop/hive/JoinedFlatTableTest.java deleted file mode 100644 index 6730f10..0000000 --- a/job/src/test/java/org/apache/kylin/job/hadoop/hive/JoinedFlatTableTest.java +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.job.hadoop.hive; - -import static org.junit.Assert.assertEquals; - -import java.io.IOException; - -import org.apache.kylin.common.KylinConfig; -import org.apache.kylin.common.util.LocalFileMetadataTestCase; -import org.apache.kylin.cube.CubeInstance; -import org.apache.kylin.cube.CubeManager; -import org.apache.kylin.cube.CubeSegment; -import org.apache.kylin.job.JoinedFlatTable; -import org.apache.kylin.job.engine.JobEngineConfig; -import org.junit.After; -import org.junit.Before; -import org.junit.Ignore; -import org.junit.Test; - -/** - * @author George Song (ysong1) - * - */ -@Ignore("This test case doesn't have much value, ignore it.") -public class JoinedFlatTableTest extends LocalFileMetadataTestCase { - - CubeInstance cube = null; - CubeJoinedFlatTableDesc intermediateTableDesc = null; - String fakeJobUUID = "abc-def"; - CubeSegment cubeSegment = null; - - @Before - public void setUp() throws Exception { - this.createTestMetadata(); - cube = CubeManager.getInstance(getTestConfig()).getCube("test_kylin_cube_with_slr_ready"); - cubeSegment = cube.getSegments().get(0); - intermediateTableDesc = new CubeJoinedFlatTableDesc(cube.getDescriptor(), cubeSegment); - } - - @After - public void after() throws Exception { - this.cleanupTestMetadata(); - } - - @Test - public void testGenCreateTableDDL() { - String ddl = JoinedFlatTable.generateCreateTableStatement(intermediateTableDesc, "/tmp", fakeJobUUID); - System.out.println(ddl); - - System.out.println("The length for the ddl is " + ddl.length()); - } - - @Test - public void testGenDropTableDDL() { - String ddl = JoinedFlatTable.generateDropTableStatement(intermediateTableDesc, fakeJobUUID); - System.out.println(ddl); - assertEquals(107, ddl.length()); - } - - @Test - public void testGenerateInsertSql() throws IOException { - String sqls = JoinedFlatTable.generateInsertDataStatement(intermediateTableDesc, fakeJobUUID, new JobEngineConfig(KylinConfig.getInstanceFromEnv())); - System.out.println(sqls); - - int length = sqls.length(); - assertEquals(1155, length); - } - -} http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/test/java/org/apache/kylin/job/impl/threadpool/BaseSchedulerTest.java ---------------------------------------------------------------------- diff --git a/job/src/test/java/org/apache/kylin/job/impl/threadpool/BaseSchedulerTest.java b/job/src/test/java/org/apache/kylin/job/impl/threadpool/BaseSchedulerTest.java deleted file mode 100644 index f3f4272..0000000 --- a/job/src/test/java/org/apache/kylin/job/impl/threadpool/BaseSchedulerTest.java +++ /dev/null @@ -1,102 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.job.impl.threadpool; - -import java.lang.reflect.Field; -import java.lang.reflect.Modifier; - -import org.apache.kylin.common.KylinConfig; -import org.apache.kylin.common.lock.ZookeeperJobLock; -import org.apache.kylin.common.util.HBaseMetadataTestCase; -import org.apache.kylin.job.constant.ExecutableConstants; -import org.apache.kylin.job.engine.JobEngineConfig; -import org.apache.kylin.job.execution.AbstractExecutable; -import org.apache.kylin.job.execution.ExecutableState; -import org.apache.kylin.job.manager.ExecutableManager; -import org.junit.After; -import org.junit.Before; - -/** - * Created by qianzhou on 12/26/14. - */ -public abstract class BaseSchedulerTest extends HBaseMetadataTestCase { - - private DefaultScheduler scheduler; - - protected ExecutableManager jobService; - - static void setFinalStatic(Field field, Object newValue) throws Exception { - field.setAccessible(true); - - Field modifiersField = Field.class.getDeclaredField("modifiers"); - modifiersField.setAccessible(true); - modifiersField.setInt(field, field.getModifiers() & ~Modifier.FINAL); - - field.set(null, newValue); - } - - protected void waitForJobFinish(String jobId) { - while (true) { - AbstractExecutable job = jobService.getJob(jobId); - final ExecutableState status = job.getStatus(); - if (status == ExecutableState.SUCCEED || status == ExecutableState.ERROR || status == ExecutableState.STOPPED || status == ExecutableState.DISCARDED) { - break; - } else { - try { - Thread.sleep(5000); - } catch (InterruptedException e) { - e.printStackTrace(); - } - } - } - } - - protected void waitForJobStatus(String jobId, ExecutableState state, long interval) { - while (true) { - AbstractExecutable job = jobService.getJob(jobId); - if (job.getStatus() == state) { - break; - } else { - try { - Thread.sleep(interval); - } catch (InterruptedException e) { - e.printStackTrace(); - } - } - } - } - - @Before - public void setup() throws Exception { - createTestMetadata(); - setFinalStatic(ExecutableConstants.class.getField("DEFAULT_SCHEDULER_INTERVAL_SECONDS"), 10); - jobService = ExecutableManager.getInstance(KylinConfig.getInstanceFromEnv()); - scheduler = DefaultScheduler.getInstance(); - scheduler.init(new JobEngineConfig(KylinConfig.getInstanceFromEnv()), new ZookeeperJobLock()); - if (!scheduler.hasStarted()) { - throw new RuntimeException("scheduler has not been started"); - } - - } - - @After - public void after() throws Exception { - cleanupTestMetadata(); - } -} http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/test/java/org/apache/kylin/job/impl/threadpool/DefaultSchedulerTest.java ---------------------------------------------------------------------- diff --git a/job/src/test/java/org/apache/kylin/job/impl/threadpool/DefaultSchedulerTest.java b/job/src/test/java/org/apache/kylin/job/impl/threadpool/DefaultSchedulerTest.java deleted file mode 100644 index 9daf87f..0000000 --- a/job/src/test/java/org/apache/kylin/job/impl/threadpool/DefaultSchedulerTest.java +++ /dev/null @@ -1,151 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.job.impl.threadpool; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.assertFalse; - -import org.apache.kylin.job.BaseTestExecutable; -import org.apache.kylin.job.ErrorTestExecutable; -import org.apache.kylin.job.FailedTestExecutable; -import org.apache.kylin.job.SelfStopExecutable; -import org.apache.kylin.job.SucceedTestExecutable; -import org.apache.kylin.job.execution.DefaultChainedExecutable; -import org.apache.kylin.job.execution.ExecutableState; -import org.junit.Test; - -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ScheduledFuture; -import java.util.concurrent.TimeUnit; - -/** - * Created by qianzhou on 12/19/14. - */ -public class DefaultSchedulerTest extends BaseSchedulerTest { - - @Test - public void testSingleTaskJob() throws Exception { - DefaultChainedExecutable job = new DefaultChainedExecutable(); - BaseTestExecutable task1 = new SucceedTestExecutable(); - job.addTask(task1); - jobService.addJob(job); - waitForJobFinish(job.getId()); - assertEquals(ExecutableState.SUCCEED, jobService.getOutput(job.getId()).getState()); - assertEquals(ExecutableState.SUCCEED, jobService.getOutput(task1.getId()).getState()); - } - - @Test - public void testSucceed() throws Exception { - DefaultChainedExecutable job = new DefaultChainedExecutable(); - BaseTestExecutable task1 = new SucceedTestExecutable(); - BaseTestExecutable task2 = new SucceedTestExecutable(); - job.addTask(task1); - job.addTask(task2); - jobService.addJob(job); - waitForJobFinish(job.getId()); - assertEquals(ExecutableState.SUCCEED, jobService.getOutput(job.getId()).getState()); - assertEquals(ExecutableState.SUCCEED, jobService.getOutput(task1.getId()).getState()); - assertEquals(ExecutableState.SUCCEED, jobService.getOutput(task2.getId()).getState()); - } - - @Test - public void testSucceedAndFailed() throws Exception { - DefaultChainedExecutable job = new DefaultChainedExecutable(); - BaseTestExecutable task1 = new SucceedTestExecutable(); - BaseTestExecutable task2 = new FailedTestExecutable(); - job.addTask(task1); - job.addTask(task2); - jobService.addJob(job); - waitForJobFinish(job.getId()); - assertEquals(ExecutableState.ERROR, jobService.getOutput(job.getId()).getState()); - assertEquals(ExecutableState.SUCCEED, jobService.getOutput(task1.getId()).getState()); - assertEquals(ExecutableState.ERROR, jobService.getOutput(task2.getId()).getState()); - } - - @Test - public void testSucceedAndError() throws Exception { - DefaultChainedExecutable job = new DefaultChainedExecutable(); - BaseTestExecutable task1 = new ErrorTestExecutable(); - BaseTestExecutable task2 = new SucceedTestExecutable(); - job.addTask(task1); - job.addTask(task2); - jobService.addJob(job); - waitForJobFinish(job.getId()); - assertEquals(ExecutableState.ERROR, jobService.getOutput(job.getId()).getState()); - assertEquals(ExecutableState.ERROR, jobService.getOutput(task1.getId()).getState()); - assertEquals(ExecutableState.READY, jobService.getOutput(task2.getId()).getState()); - } - - @Test - public void testDiscard() throws Exception { - DefaultChainedExecutable job = new DefaultChainedExecutable(); - BaseTestExecutable task1 = new SelfStopExecutable(); - job.addTask(task1); - jobService.addJob(job); - waitForJobStatus(job.getId(), ExecutableState.RUNNING, 500); - jobService.discardJob(job.getId()); - waitForJobFinish(job.getId()); - assertEquals(ExecutableState.DISCARDED, jobService.getOutput(job.getId()).getState()); - assertEquals(ExecutableState.DISCARDED, jobService.getOutput(task1.getId()).getState()); - Thread.sleep(5000); - System.out.println(job); - } - - @Test - public void testSchedulerPool() throws InterruptedException { - ScheduledExecutorService fetchPool = Executors.newScheduledThreadPool(1); - final CountDownLatch countDownLatch = new CountDownLatch(3); - ScheduledFuture future = fetchPool.scheduleAtFixedRate(new Runnable() { - @Override - public void run() { - countDownLatch.countDown(); - } - }, 5, 5, TimeUnit.SECONDS); - assertTrue("countDownLatch should reach zero in 15 secs", countDownLatch.await(20, TimeUnit.SECONDS)); - assertTrue("future should still running", future.cancel(true)); - - final CountDownLatch countDownLatch2 = new CountDownLatch(3); - ScheduledFuture future2 = fetchPool.scheduleAtFixedRate(new Runnable() { - @Override - public void run() { - countDownLatch2.countDown(); - throw new RuntimeException(); - } - }, 5, 5, TimeUnit.SECONDS); - assertFalse("countDownLatch2 should NOT reach zero in 15 secs", countDownLatch2.await(20, TimeUnit.SECONDS)); - assertFalse("future2 should has been stopped", future2.cancel(true)); - - final CountDownLatch countDownLatch3 = new CountDownLatch(3); - ScheduledFuture future3 = fetchPool.scheduleAtFixedRate(new Runnable() { - @Override - public void run() { - try { - countDownLatch3.countDown(); - throw new RuntimeException(); - } catch (Exception e) { - } - } - }, 5, 5, TimeUnit.SECONDS); - assertTrue("countDownLatch3 should reach zero in 15 secs", countDownLatch3.await(20, TimeUnit.SECONDS)); - assertTrue("future3 should still running", future3.cancel(true)); - } -} http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/test/java/org/apache/kylin/job/manager/ExecutableManagerTest.java ---------------------------------------------------------------------- diff --git a/job/src/test/java/org/apache/kylin/job/manager/ExecutableManagerTest.java b/job/src/test/java/org/apache/kylin/job/manager/ExecutableManagerTest.java deleted file mode 100644 index d6d5210..0000000 --- a/job/src/test/java/org/apache/kylin/job/manager/ExecutableManagerTest.java +++ /dev/null @@ -1,144 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.job.manager; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; - -import java.util.List; - -import org.apache.kylin.common.KylinConfig; -import org.apache.kylin.common.util.LocalFileMetadataTestCase; -import org.apache.kylin.job.BaseTestExecutable; -import org.apache.kylin.job.SucceedTestExecutable; -import org.apache.kylin.job.exception.IllegalStateTranferException; -import org.apache.kylin.job.execution.AbstractExecutable; -import org.apache.kylin.job.execution.ChainedExecutable; -import org.apache.kylin.job.execution.DefaultChainedExecutable; -import org.apache.kylin.job.execution.Executable; -import org.apache.kylin.job.execution.ExecutableState; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -/** - * Created by qianzhou on 12/16/14. - */ -public class ExecutableManagerTest extends LocalFileMetadataTestCase { - - private ExecutableManager service; - - @Before - public void setup() throws Exception { - createTestMetadata(); - service = ExecutableManager.getInstance(KylinConfig.getInstanceFromEnv()); - - for (String jobId : service.getAllJobIds()) { - System.out.println("deleting " + jobId); - service.deleteJob(jobId); - } - - } - - @After - public void after() throws Exception { - cleanupTestMetadata(); - } - - @Test - public void test() throws Exception { - assertNotNull(service); - BaseTestExecutable executable = new SucceedTestExecutable(); - executable.setParam("test1", "test1"); - executable.setParam("test2", "test2"); - executable.setParam("test3", "test3"); - service.addJob(executable); - List<AbstractExecutable> result = service.getAllExecutables(); - assertEquals(1, result.size()); - AbstractExecutable another = service.getJob(executable.getId()); - assertJobEqual(executable, another); - - service.updateJobOutput(executable.getId(), ExecutableState.RUNNING, null, "test output"); - assertJobEqual(executable, service.getJob(executable.getId())); - } - - @Test - public void testDefaultChainedExecutable() throws Exception { - DefaultChainedExecutable job = new DefaultChainedExecutable(); - job.addTask(new SucceedTestExecutable()); - job.addTask(new SucceedTestExecutable()); - - service.addJob(job); - assertEquals(2, job.getTasks().size()); - AbstractExecutable anotherJob = service.getJob(job.getId()); - assertEquals(DefaultChainedExecutable.class, anotherJob.getClass()); - assertEquals(2, ((DefaultChainedExecutable) anotherJob).getTasks().size()); - assertJobEqual(job, anotherJob); - } - - @Test - public void testValidStateTransfer() throws Exception { - SucceedTestExecutable job = new SucceedTestExecutable(); - String id = job.getId(); - service.addJob(job); - service.updateJobOutput(id, ExecutableState.RUNNING, null, null); - service.updateJobOutput(id, ExecutableState.ERROR, null, null); - service.updateJobOutput(id, ExecutableState.READY, null, null); - service.updateJobOutput(id, ExecutableState.RUNNING, null, null); - service.updateJobOutput(id, ExecutableState.READY, null, null); - service.updateJobOutput(id, ExecutableState.RUNNING, null, null); - service.updateJobOutput(id, ExecutableState.SUCCEED, null, null); - } - - @Test(expected = IllegalStateTranferException.class) - public void testInvalidStateTransfer() { - SucceedTestExecutable job = new SucceedTestExecutable(); - service.addJob(job); - service.updateJobOutput(job.getId(), ExecutableState.RUNNING, null, null); - service.updateJobOutput(job.getId(), ExecutableState.STOPPED, null, null); - } - - private static void assertJobEqual(Executable one, Executable another) { - assertEquals(one.getClass(), another.getClass()); - assertEquals(one.getId(), another.getId()); - assertEquals(one.getStatus(), another.getStatus()); - assertEquals(one.isRunnable(), another.isRunnable()); - assertEquals(one.getOutput(), another.getOutput()); - assertTrue((one.getParams() == null && another.getParams() == null) || (one.getParams() != null && another.getParams() != null)); - if (one.getParams() != null) { - assertEquals(one.getParams().size(), another.getParams().size()); - for (String key : one.getParams().keySet()) { - assertEquals(one.getParams().get(key), another.getParams().get(key)); - } - } - if (one instanceof ChainedExecutable) { - assertTrue(another instanceof ChainedExecutable); - List<? extends Executable> onesSubs = ((ChainedExecutable) one).getTasks(); - List<? extends Executable> anotherSubs = ((ChainedExecutable) another).getTasks(); - assertTrue((onesSubs == null && anotherSubs == null) || (onesSubs != null && anotherSubs != null)); - if (onesSubs != null) { - assertEquals(onesSubs.size(), anotherSubs.size()); - for (int i = 0; i < onesSubs.size(); ++i) { - assertJobEqual(onesSubs.get(i), anotherSubs.get(i)); - } - } - } - } -} http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/test/java/org/apache/kylin/job/tools/ColumnCardinalityJobTest.java ---------------------------------------------------------------------- diff --git a/job/src/test/java/org/apache/kylin/job/tools/ColumnCardinalityJobTest.java b/job/src/test/java/org/apache/kylin/job/tools/ColumnCardinalityJobTest.java deleted file mode 100644 index 2c2f16f..0000000 --- a/job/src/test/java/org/apache/kylin/job/tools/ColumnCardinalityJobTest.java +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.job.tools; - -import static org.junit.Assert.assertEquals; - -import java.io.File; -import java.io.IOException; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileUtil; -import org.apache.hadoop.util.ToolRunner; -import org.apache.kylin.job.hadoop.cardinality.HiveColumnCardinalityJob; -import org.apache.kylin.job.hadoop.cube.BaseCuboidJobTest; -import org.junit.Before; -import org.junit.Ignore; -import org.junit.Test; - -/** - * @author ysong1 - * - */ -@Ignore("This test is invalid now as the mapper uses HCatalog to fetch the data which need a hive env") -public class ColumnCardinalityJobTest { - - private Configuration conf; - - @Before - public void setup() throws IOException { - conf = new Configuration(); - BaseCuboidJobTest.setLocalMR(conf); - } - - @Test - @Ignore("not maintaining") - public void testJob() throws Exception { - final String input = "src/test/resources/data/test_cal_dt/"; - final String output = "target/test-output/column-cardinality/"; - - FileUtil.fullyDelete(new File(output)); - - String[] args = { "-input", input, "-output", output, "-cols", "1,2,3,4,5,6,9,0" }; - assertEquals("Job failed", 0, ToolRunner.run(new HiveColumnCardinalityJob(), args)); - } - -} http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/test/java/org/apache/kylin/job/tools/ColumnCardinalityMapperTest.java ---------------------------------------------------------------------- diff --git a/job/src/test/java/org/apache/kylin/job/tools/ColumnCardinalityMapperTest.java b/job/src/test/java/org/apache/kylin/job/tools/ColumnCardinalityMapperTest.java deleted file mode 100644 index 0d96156..0000000 --- a/job/src/test/java/org/apache/kylin/job/tools/ColumnCardinalityMapperTest.java +++ /dev/null @@ -1,126 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.job.tools; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -import java.io.BufferedReader; -import java.io.File; -import java.io.FileReader; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.List; - -import org.apache.hadoop.io.BytesWritable; -import org.apache.hadoop.io.IntWritable; -import org.apache.hadoop.io.LongWritable; -import org.apache.hadoop.io.Text; -import org.apache.hadoop.mrunit.mapreduce.MapDriver; -import org.apache.hadoop.mrunit.types.Pair; -import org.apache.kylin.common.hll.HyperLogLogPlusCounter; -import org.apache.kylin.job.hadoop.cardinality.ColumnCardinalityMapper; -import org.junit.Before; -import org.junit.Ignore; -import org.junit.Test; - -/** - * @author ysong1 - * - */ -@Ignore("This test is invalid now as the mapper uses HCatalog to fetch the data which need a hive env") -public class ColumnCardinalityMapperTest { - - @SuppressWarnings("rawtypes") - MapDriver mapDriver; - String localTempDir = System.getProperty("java.io.tmpdir") + File.separator; - - @SuppressWarnings({ "rawtypes", "unchecked" }) - @Before - public void setUp() { - ColumnCardinalityMapper mapper = new ColumnCardinalityMapper(); - mapDriver = MapDriver.newMapDriver(mapper); - } - - public final static String strArr = "abc,tests,test,test,as,sts,test,tss,sets"; - - @SuppressWarnings({ "unchecked" }) - @Test - @Ignore - public void testMapperOn177() throws IOException { - mapDriver.clearInput(); - File file = new File("src/test/resources/data/test_cal_dt/part-r-00000"); - FileReader reader = new FileReader(file); - BufferedReader breader = new BufferedReader(reader); - String s = breader.readLine(); - int i = 0; - while (s != null) { - LongWritable inputKey = new LongWritable(i++); - mapDriver.addInput(inputKey, new Text(s)); - s = breader.readLine(); - } - // breader.close(); - List<Pair<IntWritable, BytesWritable>> result = mapDriver.run(); - breader.close(); - assertEquals(9, result.size()); - - int key1 = result.get(0).getFirst().get(); - BytesWritable value1 = result.get(0).getSecond(); - byte[] bytes = value1.getBytes(); - HyperLogLogPlusCounter hllc = new HyperLogLogPlusCounter(); - hllc.readRegisters(ByteBuffer.wrap(bytes)); - assertTrue(key1 > 0); - assertEquals(8, hllc.getCountEstimate()); - } - - @SuppressWarnings("unchecked") - @Test - public void testMapperOnComma() throws IOException { - mapDriver.clearInput(); - LongWritable inputKey1 = new LongWritable(1); - LongWritable inputKey2 = new LongWritable(2); - LongWritable inputKey3 = new LongWritable(3); - LongWritable inputKey4 = new LongWritable(4); - LongWritable inputKey5 = new LongWritable(5); - LongWritable inputKey6 = new LongWritable(6); - LongWritable inputKey7 = new LongWritable(7); - - mapDriver.addInput(inputKey1, new Text()); - mapDriver.addInput(inputKey2, new Text(strArr)); - mapDriver.addInput(inputKey3, new Text(strArr)); - mapDriver.addInput(inputKey4, new Text(strArr)); - mapDriver.addInput(inputKey5, new Text(strArr)); - mapDriver.addInput(inputKey6, new Text(strArr)); - mapDriver.addInput(inputKey7, new Text(strArr)); - - List<Pair<IntWritable, BytesWritable>> result = mapDriver.run(); - - assertEquals(9, result.size()); - - int key1 = result.get(0).getFirst().get(); - BytesWritable value1 = result.get(0).getSecond(); - byte[] bytes = value1.getBytes(); - HyperLogLogPlusCounter hllc = new HyperLogLogPlusCounter(); - hllc.readRegisters(ByteBuffer.wrap(bytes)); - System.out.println("ab\177ab".length()); - assertTrue(key1 > 0); - assertEquals(1, hllc.getCountEstimate()); - } - -} http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/test/java/org/apache/kylin/job/tools/ColumnCardinalityReducerTest.java ---------------------------------------------------------------------- diff --git a/job/src/test/java/org/apache/kylin/job/tools/ColumnCardinalityReducerTest.java b/job/src/test/java/org/apache/kylin/job/tools/ColumnCardinalityReducerTest.java deleted file mode 100644 index 15d1ffb..0000000 --- a/job/src/test/java/org/apache/kylin/job/tools/ColumnCardinalityReducerTest.java +++ /dev/null @@ -1,113 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.job.tools; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -import java.io.File; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.List; -import java.util.StringTokenizer; - -import org.apache.hadoop.io.BytesWritable; -import org.apache.hadoop.io.IntWritable; -import org.apache.hadoop.io.LongWritable; -import org.apache.hadoop.mrunit.mapreduce.ReduceDriver; -import org.apache.hadoop.mrunit.types.Pair; -import org.apache.kylin.common.hll.HyperLogLogPlusCounter; -import org.apache.kylin.common.util.Bytes; -import org.apache.kylin.cube.kv.RowConstants; -import org.apache.kylin.job.hadoop.cardinality.ColumnCardinalityMapper; -import org.apache.kylin.job.hadoop.cardinality.ColumnCardinalityReducer; -import org.junit.Before; -import org.junit.Test; - -/** - * @author ysong1 - * - */ -public class ColumnCardinalityReducerTest { - - ReduceDriver<IntWritable, BytesWritable, IntWritable, LongWritable> reduceDriver; - String localTempDir = System.getProperty("java.io.tmpdir") + File.separator; - - @Before - public void setUp() { - ColumnCardinalityReducer reducer = new ColumnCardinalityReducer(); - reduceDriver = ReduceDriver.newReduceDriver(reducer); - } - - private byte[] getBytes(String str) throws IOException { - HyperLogLogPlusCounter hllc = new HyperLogLogPlusCounter(); - StringTokenizer tokenizer = new StringTokenizer(str, ColumnCardinalityMapper.DEFAULT_DELIM); - int i = 0; - while (tokenizer.hasMoreTokens()) { - String temp = i + "_" + tokenizer.nextToken(); - i++; - hllc.add(Bytes.toBytes(temp)); - } - ByteBuffer buf = ByteBuffer.allocate(RowConstants.ROWVALUE_BUFFER_SIZE); - buf.clear(); - hllc.writeRegisters(buf); - buf.flip(); - return buf.array(); - } - - @Test - public void testReducer() throws IOException { - IntWritable key1 = new IntWritable(1); - List<BytesWritable> values1 = new ArrayList<BytesWritable>(); - values1.add(new BytesWritable(getBytes(ColumnCardinalityMapperTest.strArr))); - - IntWritable key2 = new IntWritable(2); - List<BytesWritable> values2 = new ArrayList<BytesWritable>(); - values2.add(new BytesWritable(getBytes(ColumnCardinalityMapperTest.strArr + " x"))); - - IntWritable key3 = new IntWritable(3); - List<BytesWritable> values3 = new ArrayList<BytesWritable>(); - values3.add(new BytesWritable(getBytes(ColumnCardinalityMapperTest.strArr + " xx"))); - - IntWritable key4 = new IntWritable(4); - List<BytesWritable> values4 = new ArrayList<BytesWritable>(); - values4.add(new BytesWritable(getBytes(ColumnCardinalityMapperTest.strArr + " xxx"))); - - IntWritable key5 = new IntWritable(5); - List<BytesWritable> values5 = new ArrayList<BytesWritable>(); - values5.add(new BytesWritable(getBytes(ColumnCardinalityMapperTest.strArr + " xxxx"))); - - reduceDriver.withInput(key1, values1); - reduceDriver.withInput(key2, values2); - reduceDriver.withInput(key3, values3); - reduceDriver.withInput(key4, values4); - reduceDriver.withInput(key5, values5); - - List<Pair<IntWritable, LongWritable>> result = reduceDriver.run(); - - assertEquals(5, result.size()); - - int outputKey1 = result.get(0).getFirst().get(); - LongWritable value1 = result.get(0).getSecond(); - assertTrue(outputKey1 == 1); - assertTrue((10 == value1.get()) || (9 == value1.get())); - - } -} http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/test/java/org/apache/kylin/job/tools/CubeMigrationTests.java ---------------------------------------------------------------------- diff --git a/job/src/test/java/org/apache/kylin/job/tools/CubeMigrationTests.java b/job/src/test/java/org/apache/kylin/job/tools/CubeMigrationTests.java deleted file mode 100644 index 61f44d1..0000000 --- a/job/src/test/java/org/apache/kylin/job/tools/CubeMigrationTests.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.job.tools; - -import java.io.File; -import java.io.IOException; - -import org.apache.kylin.common.util.AbstractKylinTestCase; -import org.apache.kylin.common.util.ClassUtil; -import org.apache.kylin.common.util.LocalFileMetadataTestCase; -import org.codehaus.jettison.json.JSONException; -import org.junit.After; -import org.junit.Before; -import org.junit.Ignore; -import org.junit.Test; - -/** - * Created by honma on 9/17/14. - */ -@Ignore("convenient trial tool for dev") -public class CubeMigrationTests extends LocalFileMetadataTestCase { - @Before - public void setup() throws Exception { - super.createTestMetadata(); - ClassUtil.addClasspath(new File(AbstractKylinTestCase.SANDBOX_TEST_DATA).getAbsolutePath()); - } - - @After - public void clean() { - this.cleanupTestMetadata(); - } - - @Test - public void testMigrate() throws IOException, JSONException, InterruptedException { - - // CubeMigrationCLI.moveCube(KylinConfig.getInstanceFromEnv(), - // KylinConfig.getInstanceFromEnv(), - // "test_kylin_cube_with_slr_empty", "migration", "true", "false"); - } - -} http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/test/java/org/apache/kylin/job/tools/HBaseRowDigestTest.java ---------------------------------------------------------------------- diff --git a/job/src/test/java/org/apache/kylin/job/tools/HBaseRowDigestTest.java b/job/src/test/java/org/apache/kylin/job/tools/HBaseRowDigestTest.java deleted file mode 100644 index 9f9c23c..0000000 --- a/job/src/test/java/org/apache/kylin/job/tools/HBaseRowDigestTest.java +++ /dev/null @@ -1,103 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.job.tools; - -import java.io.File; -import java.io.IOException; - -import org.apache.commons.io.FileUtils; -import org.apache.hadoop.hbase.Cell; -import org.apache.hadoop.hbase.client.HConnection; -import org.apache.hadoop.hbase.client.HTableInterface; -import org.apache.hadoop.hbase.client.Result; -import org.apache.hadoop.hbase.client.ResultScanner; -import org.apache.hadoop.hbase.io.ImmutableBytesWritable; -import org.apache.kylin.common.persistence.HBaseConnection; -import org.apache.kylin.common.util.BytesUtil; -import org.apache.kylin.common.util.HBaseMetadataTestCase; -import org.junit.After; -import org.junit.Before; -import org.junit.Ignore; -import org.junit.Test; - -/** - * Created by Hongbin Ma(Binmahone) on 2/6/15. - */ -@Ignore -public class HBaseRowDigestTest extends HBaseMetadataTestCase { - - @Before - public void setUp() throws Exception { - this.createTestMetadata(); - } - - @After - public void tearDown() throws Exception { - this.cleanupTestMetadata(); - } - - private static final byte[] CF = "f".getBytes(); - private static final byte[] QN = "c".getBytes(); - static ImmutableBytesWritable k = new ImmutableBytesWritable(); - static ImmutableBytesWritable v = new ImmutableBytesWritable(); - - @Test - public static void test() throws IOException { - String hbaseUrl = "hbase"; // use hbase-site.xml on classpath - HConnection conn = null; - HTableInterface table = null; - try { - conn = HBaseConnection.get(hbaseUrl); - table = conn.getTable("KYLIN_II_YTYWP3CQGJ"); - ResultScanner scanner = table.getScanner(CF, QN); - StringBuffer sb = new StringBuffer(); - while (true) { - Result r = scanner.next(); - if (r == null) - break; - - Cell[] cells = r.rawCells(); - Cell c = cells[0]; - - k.set(c.getRowArray(), c.getRowOffset(), c.getRowLength()); - v.set(c.getValueArray(), c.getValueOffset(), c.getValueLength()); - - byte[] row = k.copyBytes(); - byte[] value = v.copyBytes(); - // byte[] row = r.getRow(); - // byte[] value = r.getValue(CF, QN); - // - sb.append("row length: " + row.length + "\r\n"); - sb.append(BytesUtil.toReadableText(row) + "\r\n"); - sb.append("value length: " + value.length + "\r\n"); - sb.append(BytesUtil.toReadableText(value) + "\r\n"); - } - System.out.println(sb.toString()); - FileUtils.writeStringToFile(new File("/Users/honma/Desktop/a3"), sb.toString()); - } catch (IOException e) { - e.printStackTrace(); - } finally { - if (table != null) - table.close(); - if (conn != null) - conn.close(); - } - - } -} http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/test/resources/data/8d_cuboid/part-r-00000 ---------------------------------------------------------------------- diff --git a/job/src/test/resources/data/8d_cuboid/part-r-00000 b/job/src/test/resources/data/8d_cuboid/part-r-00000 deleted file mode 100644 index d277125..0000000 Binary files a/job/src/test/resources/data/8d_cuboid/part-r-00000 and /dev/null differ http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/test/resources/data/base_cuboid/part-r-00000 ---------------------------------------------------------------------- diff --git a/job/src/test/resources/data/base_cuboid/part-r-00000 b/job/src/test/resources/data/base_cuboid/part-r-00000 deleted file mode 100644 index ed53ffb..0000000 Binary files a/job/src/test/resources/data/base_cuboid/part-r-00000 and /dev/null differ http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/job/src/test/resources/data/flat_table/000000_0 ---------------------------------------------------------------------- diff --git a/job/src/test/resources/data/flat_table/000000_0 b/job/src/test/resources/data/flat_table/000000_0 deleted file mode 100644 index 8b1b7cc..0000000 Binary files a/job/src/test/resources/data/flat_table/000000_0 and /dev/null differ
