Updated Branches: refs/heads/master 91ffb14bc -> 07059ec04
BIGTOP-920: Add HCatalog tests Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/07059ec0 Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/07059ec0 Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/07059ec0 Branch: refs/heads/master Commit: 07059ec04539b265943732a1b799db3e643c9f21 Parents: 91ffb14 Author: Mark Grover <[email protected]> Authored: Mon May 6 17:16:45 2013 -0700 Committer: Roman Shaposhnik <[email protected]> Committed: Mon May 6 17:27:27 2013 -0700 ---------------------------------------------------------------------- bigtop-tests/test-artifacts/hcatalog/README | 16 ++ bigtop-tests/test-artifacts/hcatalog/pom.xml | 32 ++++ .../itest/hcatalogsmoke/TestHcatalogBasic.groovy | 145 +++++++++++++++ .../src/main/resources/data/data-2013-01-01.txt | 4 + .../src/main/resources/data/data-2013-01-02.txt | 3 + .../src/main/resources/hcat_basic_count.expected | 1 + .../main/resources/hcat_basic_describe.expected | 3 + .../main/resources/hcat_basic_partitions.expected | 2 + bigtop-tests/test-artifacts/pom.xml | 1 + .../test-execution/smokes/hcatalog/pom.xml | 83 ++++++++ bigtop-tests/test-execution/smokes/pom.xml | 8 + 11 files changed, 298 insertions(+), 0 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/bigtop/blob/07059ec0/bigtop-tests/test-artifacts/hcatalog/README ---------------------------------------------------------------------- diff --git a/bigtop-tests/test-artifacts/hcatalog/README b/bigtop-tests/test-artifacts/hcatalog/README new file mode 100644 index 0000000..03ef988 --- /dev/null +++ b/bigtop-tests/test-artifacts/hcatalog/README @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +This is a project to develop and build HCatalog smoke and system tests. http://git-wip-us.apache.org/repos/asf/bigtop/blob/07059ec0/bigtop-tests/test-artifacts/hcatalog/pom.xml ---------------------------------------------------------------------- diff --git a/bigtop-tests/test-artifacts/hcatalog/pom.xml b/bigtop-tests/test-artifacts/hcatalog/pom.xml new file mode 100644 index 0000000..7c8c83b --- /dev/null +++ b/bigtop-tests/test-artifacts/hcatalog/pom.xml @@ -0,0 +1,32 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + + <parent> + <groupId>org.apache.bigtop.itest</groupId> + <artifactId>bigtop-smokes</artifactId> + <version>0.6.0-SNAPSHOT</version> + <relativePath>../pom.xml</relativePath> + </parent> + + <groupId>org.apache.bigtop.itest</groupId> + <artifactId>hcatalog-smoke</artifactId> + <version>0.6.0-SNAPSHOT</version> + <name>hcatalogsmoke</name> +</project> http://git-wip-us.apache.org/repos/asf/bigtop/blob/07059ec0/bigtop-tests/test-artifacts/hcatalog/src/main/groovy/org/apache/bigtop/itest/hcatalogsmoke/TestHcatalogBasic.groovy ---------------------------------------------------------------------- diff --git a/bigtop-tests/test-artifacts/hcatalog/src/main/groovy/org/apache/bigtop/itest/hcatalogsmoke/TestHcatalogBasic.groovy b/bigtop-tests/test-artifacts/hcatalog/src/main/groovy/org/apache/bigtop/itest/hcatalogsmoke/TestHcatalogBasic.groovy new file mode 100644 index 0000000..ca609c7 --- /dev/null +++ b/bigtop-tests/test-artifacts/hcatalog/src/main/groovy/org/apache/bigtop/itest/hcatalogsmoke/TestHcatalogBasic.groovy @@ -0,0 +1,145 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.bigtop.itest.hcatalogsmoke + +import java.util.ArrayList +import java.util.List +import java.util.Date + +import org.junit.After +import org.junit.AfterClass +import org.junit.Before +import org.junit.BeforeClass +import org.junit.Test +import static org.junit.Assert.assertTrue +import static org.junit.Assert.assertEquals + +import org.apache.bigtop.itest.junit.OrderedParameterized +import org.junit.runner.RunWith +import org.junit.runners.Parameterized.Parameters +import org.apache.bigtop.itest.Contract +import org.apache.bigtop.itest.ParameterSetter +import org.apache.bigtop.itest.Property +import org.apache.bigtop.itest.shell.Shell + +@RunWith(OrderedParameterized.class) +public class TestHcatalogBasic { + + public static Shell sh = new Shell("/bin/bash -sx") + + public TestHcatalogBasic() { + } + + + @BeforeClass + public static void setUp() { + } + + @AfterClass + public static void tearDown() { + sh.exec("rm -f *.actual") + sh.exec("hive -e \"DROP TABLE IF EXISTS hcat_basic\"") + sh.exec("hadoop fs -rmr -skipTrash /user/hive/warehouse") + } + + + /** + * Validate that the table created via hcat exists from Hive's world view + */ + @Test + public void testBasic() { + sh.exec(""" + hcat -e "CREATE TABLE hcat_basic(key string, value string) \ + PARTITIONED BY (dt STRING) \ + ROW FORMAT DELIMITED FIELDS TERMINATED BY ','" + """) + assertTrue("Could not create table via hcat, return code: " + sh.ret, sh.ret == 0) + + sh.exec(""" + hive -e "DESCRIBE hcat_basic" > hive_hcat_basic_verify.actual + diff -u hcat_basic_describe.expected hive_hcat_basic_verify.actual + """) + assertEquals("hive couldn't detect the table created via hcat, return code: " + sh.ret, + 0, sh.ret); + + sh.exec(""" + hcat -e "DESCRIBE hcat_basic" > hcat_hcat_basic_verify.actual + diff -u hcat_basic_describe.expected hcat_hcat_basic_verify.actual + """) + assertEquals("hcat couldn't detect the table created via hcat, return code: " + sh.ret, + 0, sh.ret); + + // Add a partition via hive + sh.exec("hive -e \"ALTER TABLE hcat_basic ADD PARTITION (dt='2013-01-01')\"") + // Add another partition via hcat + sh.exec("hcat -e \"ALTER TABLE hcat_basic ADD PARTITION (dt='2013-01-02')\"") + + sh.exec(""" + hive -e "SHOW PARTITIONS hcat_basic" > hive_hcat_basic_partitions.actual + diff -u hcat_basic_partitions.expected hive_hcat_basic_partitions.actual + """) + assertEquals("hive couldn't detect all the partitions of the table, return code: " + sh.ret, 0, sh.ret) + + sh.exec(""" + hcat -e "SHOW PARTITIONS hcat_basic" > hcat_hcat_basic_partitions.actual + diff -u hcat_basic_partitions.expected hcat_hcat_basic_partitions.actual + """) + assertEquals("hcat couldn't detect all the partitions of the table, return code: " + sh.ret, 0, sh.ret) + + // Load data into various partitions of the table + sh.exec(""" + hive -e "LOAD DATA LOCAL INPATH 'data/data-2013-01-01.txt' OVERWRITE INTO TABLE hcat_basic PARTITION(dt='2013-01-01')" + hive -e "LOAD DATA LOCAL INPATH 'data/data-2013-01-02.txt' OVERWRITE INTO TABLE hcat_basic PARTITION(dt='2013-01-02')" + """) + assertEquals("Error in loading data in table, return code: " + sh.ret, 0, sh.ret) + + // Count the number of records via hive + sh.exec(""" + hive -e "SELECT COUNT(*) FROM hcat_basic" > hive_hcat_basic_count.actual + diff -u hcat_basic_count.expected hive_hcat_basic_count.actual + """) + assertEquals("hive's count of records doesn't match expected count, return code: " + sh.ret, 0, sh.ret) + + // Test Pig's integration with HCatalog + sh.exec(""" + pig -useHCatalog -e "\ + REGISTER /usr/lib/hcatalog/share/hcatalog/*.jar; \ + REGISTER /usr/lib/hive/lib/*.jar; \ + DATA= LOAD 'hcat_basic' USING org.apache.hcatalog.pig.HCatLoader(); \ + DATA_GROUPS= GROUP DATA ALL; \ + DATA_COUNT= FOREACH DATA_GROUPS GENERATE COUNT(DATA); \ + DUMP DATA_COUNT;" > pig_hcat_basic_count.actual + diff hcat_basic_count.expected <(cat pig_hcat_basic_count.actual | sed -e 's/(//g' -e 's/)//g') + """) + assertEquals("pig's count of records doesn't match expected count, return code: " + sh.ret, 0, sh.ret) + + sh.exec(""" + hcat -e "DROP TABLE hcat_basic" + """) + assertEquals("hcat wasn't able to drop table hcat_basic, return code: " + sh.ret, 0, sh.ret) + + } + + @Parameters + public static Map<String, Object[]> readTestCases() { + Map<String, Object[]> result = new HashMap<String, Object[]>() + result.put(new String(), new Object[0]) + return result; + } +} http://git-wip-us.apache.org/repos/asf/bigtop/blob/07059ec0/bigtop-tests/test-artifacts/hcatalog/src/main/resources/data/data-2013-01-01.txt ---------------------------------------------------------------------- diff --git a/bigtop-tests/test-artifacts/hcatalog/src/main/resources/data/data-2013-01-01.txt b/bigtop-tests/test-artifacts/hcatalog/src/main/resources/data/data-2013-01-01.txt new file mode 100644 index 0000000..d78890f --- /dev/null +++ b/bigtop-tests/test-artifacts/hcatalog/src/main/resources/data/data-2013-01-01.txt @@ -0,0 +1,4 @@ +k,v1 +k,v2 +k,v3 +k,v4 http://git-wip-us.apache.org/repos/asf/bigtop/blob/07059ec0/bigtop-tests/test-artifacts/hcatalog/src/main/resources/data/data-2013-01-02.txt ---------------------------------------------------------------------- diff --git a/bigtop-tests/test-artifacts/hcatalog/src/main/resources/data/data-2013-01-02.txt b/bigtop-tests/test-artifacts/hcatalog/src/main/resources/data/data-2013-01-02.txt new file mode 100644 index 0000000..c51cb55 --- /dev/null +++ b/bigtop-tests/test-artifacts/hcatalog/src/main/resources/data/data-2013-01-02.txt @@ -0,0 +1,3 @@ +k',v1 +k',v2 +k',v5 http://git-wip-us.apache.org/repos/asf/bigtop/blob/07059ec0/bigtop-tests/test-artifacts/hcatalog/src/main/resources/hcat_basic_count.expected ---------------------------------------------------------------------- diff --git a/bigtop-tests/test-artifacts/hcatalog/src/main/resources/hcat_basic_count.expected b/bigtop-tests/test-artifacts/hcatalog/src/main/resources/hcat_basic_count.expected new file mode 100644 index 0000000..7f8f011 --- /dev/null +++ b/bigtop-tests/test-artifacts/hcatalog/src/main/resources/hcat_basic_count.expected @@ -0,0 +1 @@ +7 http://git-wip-us.apache.org/repos/asf/bigtop/blob/07059ec0/bigtop-tests/test-artifacts/hcatalog/src/main/resources/hcat_basic_describe.expected ---------------------------------------------------------------------- diff --git a/bigtop-tests/test-artifacts/hcatalog/src/main/resources/hcat_basic_describe.expected b/bigtop-tests/test-artifacts/hcatalog/src/main/resources/hcat_basic_describe.expected new file mode 100644 index 0000000..7c74ac2 --- /dev/null +++ b/bigtop-tests/test-artifacts/hcatalog/src/main/resources/hcat_basic_describe.expected @@ -0,0 +1,3 @@ +key string +value string +dt string http://git-wip-us.apache.org/repos/asf/bigtop/blob/07059ec0/bigtop-tests/test-artifacts/hcatalog/src/main/resources/hcat_basic_partitions.expected ---------------------------------------------------------------------- diff --git a/bigtop-tests/test-artifacts/hcatalog/src/main/resources/hcat_basic_partitions.expected b/bigtop-tests/test-artifacts/hcatalog/src/main/resources/hcat_basic_partitions.expected new file mode 100644 index 0000000..c4e0be9 --- /dev/null +++ b/bigtop-tests/test-artifacts/hcatalog/src/main/resources/hcat_basic_partitions.expected @@ -0,0 +1,2 @@ +dt=2013-01-01 +dt=2013-01-02 http://git-wip-us.apache.org/repos/asf/bigtop/blob/07059ec0/bigtop-tests/test-artifacts/pom.xml ---------------------------------------------------------------------- diff --git a/bigtop-tests/test-artifacts/pom.xml b/bigtop-tests/test-artifacts/pom.xml index ed05bae..428ce24 100644 --- a/bigtop-tests/test-artifacts/pom.xml +++ b/bigtop-tests/test-artifacts/pom.xml @@ -47,6 +47,7 @@ <module>crunch</module> <module>datafu</module> <module>fatjar</module> + <module>hcatalog</module> </modules> <dependencies> http://git-wip-us.apache.org/repos/asf/bigtop/blob/07059ec0/bigtop-tests/test-execution/smokes/hcatalog/pom.xml ---------------------------------------------------------------------- diff --git a/bigtop-tests/test-execution/smokes/hcatalog/pom.xml b/bigtop-tests/test-execution/smokes/hcatalog/pom.xml new file mode 100644 index 0000000..1a88989 --- /dev/null +++ b/bigtop-tests/test-execution/smokes/hcatalog/pom.xml @@ -0,0 +1,83 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<project xmlns="http://maven.apache.org/POM/4.0.0"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + + <parent> + <groupId>org.apache.bigtop.itest</groupId> + <artifactId>smoke-tests</artifactId> + <version>0.6.0-SNAPSHOT</version> + <relativePath>../pom.xml</relativePath> + </parent> + + <modelVersion>4.0.0</modelVersion> + <groupId>org.apache.bigtop.itest</groupId> + <artifactId>hcatalog-smoke-execution</artifactId> + <version>0.6.0-SNAPSHOT</version> + <name>HCatalog smoke test execution</name> + + <properties> + <org.apache.maven-dependency-plugin.groupId>org.apache.bigtop.itest</org.apache.maven-dependency-plugin.groupId> + <org.apache.maven-dependency-plugin.artifactId>hcatalog-smoke</org.apache.maven-dependency-plugin.artifactId> + <org.apache.maven-dependency-plugin.version>${hcatalog-smoke.version}</org.apache.maven-dependency-plugin.version> + <org.apache.maven-dependency-plugin.type>jar</org.apache.maven-dependency-plugin.type> + </properties> + + <dependencies> + <dependency> + <groupId>org.apache.bigtop.itest</groupId> + <artifactId>hcatalog-smoke</artifactId> + </dependency> + </dependencies> + + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-dependency-plugin</artifactId> + </plugin> + + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-enforcer-plugin</artifactId> + <version>1.0</version> + <executions> + <execution> + <id>enforce-property</id> + <goals> + <goal>enforce</goal> + </goals> + <configuration> + <fail>true</fail> + </configuration> + </execution> + </executions> + </plugin> + + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-failsafe-plugin</artifactId> + <version>2.11</version> + <configuration> + <forkMode>always</forkMode> + </configuration> + </plugin> + </plugins> + </build> +</project> http://git-wip-us.apache.org/repos/asf/bigtop/blob/07059ec0/bigtop-tests/test-execution/smokes/pom.xml ---------------------------------------------------------------------- diff --git a/bigtop-tests/test-execution/smokes/pom.xml b/bigtop-tests/test-execution/smokes/pom.xml index f8604b7..b5e97cf 100644 --- a/bigtop-tests/test-execution/smokes/pom.xml +++ b/bigtop-tests/test-execution/smokes/pom.xml @@ -47,6 +47,7 @@ <hue-smoke.version>${project.version}</hue-smoke.version> <package-smoke.version>${project.version}</package-smoke.version> <solr-smoke.version>${project.version}</solr-smoke.version> + <hcatalog-smoke.version>${project.version}</hcatalog-smoke.version> </properties> <!-- @@ -67,6 +68,7 @@ <module>flume</module> <module>sqoop</module> --> + <module>hcatalog</module> </modules> <dependencyManagement> @@ -138,6 +140,12 @@ <version>${package-smoke.version}</version> <scope>test</scope> </dependency> + <dependency> + <groupId>org.apache.bigtop.itest</groupId> + <artifactId>hcatalog-smoke</artifactId> + <version>${hcatalog-smoke.version}</version> + <scope>test</scope> + </dependency> </dependencies> </dependencyManagement>
