BIGTOP-1188. Smoke tests for HttpFs
Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/69f7c942 Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/69f7c942 Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/69f7c942 Branch: refs/heads/master Commit: 69f7c9426c4a80dde7c9b1509120cd8600c65976 Parents: d4504ae Author: Adam Kawa <[email protected]> Authored: Tue Jan 21 09:37:55 2014 +0000 Committer: vagrant <vagrant@ubuntu-server-12042-x64-vbox4210.(none)> Committed: Thu Feb 27 19:26:09 2014 +0000 ---------------------------------------------------------------------- bigtop-tests/test-artifacts/httpfs/pom.xml | 35 +++++ .../bigtop/itest/httpfs/TestHttpFs.groovy | 153 +++++++++++++++++++ .../main/resources/text-files/helloworld.txt | 1 + bigtop-tests/test-artifacts/pom.xml | 1 + .../test-execution/smokes/httpfs/pom.xml | 97 ++++++++++++ bigtop-tests/test-execution/smokes/pom.xml | 9 +- 6 files changed, 295 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/bigtop/blob/69f7c942/bigtop-tests/test-artifacts/httpfs/pom.xml ---------------------------------------------------------------------- diff --git a/bigtop-tests/test-artifacts/httpfs/pom.xml b/bigtop-tests/test-artifacts/httpfs/pom.xml new file mode 100644 index 0000000..aafa8f3 --- /dev/null +++ b/bigtop-tests/test-artifacts/httpfs/pom.xml @@ -0,0 +1,35 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<project xmlns="http://maven.apache.org/POM/4.0.0"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>org.apache.bigtop.itest</groupId> + <artifactId>bigtop-smokes</artifactId> + <version>0.8.0-SNAPSHOT</version> + <relativePath>../pom.xml</relativePath> + </parent> + + <groupId>org.apache.bigtop.itest</groupId> + <artifactId>httpfs-smoke</artifactId> + <version>0.8.0-SNAPSHOT</version> + <name>httpfssmokes</name> + +</project> http://git-wip-us.apache.org/repos/asf/bigtop/blob/69f7c942/bigtop-tests/test-artifacts/httpfs/src/main/groovy/org/apache/bigtop/itest/httpfs/TestHttpFs.groovy ---------------------------------------------------------------------- diff --git a/bigtop-tests/test-artifacts/httpfs/src/main/groovy/org/apache/bigtop/itest/httpfs/TestHttpFs.groovy b/bigtop-tests/test-artifacts/httpfs/src/main/groovy/org/apache/bigtop/itest/httpfs/TestHttpFs.groovy new file mode 100644 index 0000000..0aa9a45 --- /dev/null +++ b/bigtop-tests/test-artifacts/httpfs/src/main/groovy/org/apache/bigtop/itest/httpfs/TestHttpFs.groovy @@ -0,0 +1,153 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.bigtop.itest.hadoop.hdfs + +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.apache.bigtop.itest.shell.Shell; +import org.apache.bigtop.itest.TestUtils +import org.apache.commons.logging.Log +import org.apache.commons.logging.LogFactory + +public class TestHttpFs { + private static Log LOG = LogFactory.getLog(TestHttpFs.class) + + private static final String USERNAME = System.getProperty("user.name"); + private static final String HTTPFS_PROXY = System.getenv('HTTPFS_PROXY'); + static { + assertNotNull("HTTPFS_PROXY has to be set to run this test", + HTTPFS_PROXY); + } + + private static final String HTTPFS_PREFIX = "http://$HTTPFS_PROXY/webhdfs/v1";; + private static final String HTTPFS_SUCCESS = "{\"boolean\":true}"; + + private static final String DATA_DIR = System.getProperty("data.dir", "text-files"); + + private static String testHttpFsFolder = "/tmp/httpfssmoke-" + (new Date().getTime()); + private static String testHttpFsFolderRenamed = "$testHttpFsFolder-renamed"; + + private static Shell shHDFS = new Shell("/bin/bash", USERNAME); + private static Shell sh = new Shell("/bin/bash"); + + + @BeforeClass + public static void setUp() { + } + + @AfterClass + public static void tearDown() { + // clean up of existing folders + shHDFS.exec("hadoop fs -test -e $testHttpFsFolder"); + if (shHDFS.getRet() == 0) { + shHDFS.exec("hadoop fs -rmr -skipTrash $testHttpFsFolder"); + assertTrue("Deletion of previous testHttpFsFolder from HDFS failed", + shHDFS.getRet() == 0); + } + shHDFS.exec("hadoop fs -test -e $testHttpFsFolderRenamed"); + if (shHDFS.getRet() == 0) { + shHDFS.exec("hadoop fs -rmr -skipTrash $testHttpFsFolderRenamed"); + assertTrue("Deletion of previous testHttpFsFolderRenamed from HDFS failed", + shHDFS.getRet() == 0); + } + } + + public void assertValueExists(List<String> values, String expected) { + boolean exists = false; + for (String value: values) { + if (expected.startsWith(value)) { + exists = true; + } + } + assertTrue(expected + " NOT found!", exists == true); + } + + private void createDir(String dirname) { + sh.exec("curl -i -X PUT '$HTTPFS_PREFIX$dirname?user.name=$USERNAME&op=MKDIRS'"); + } + + @Test + public void testCreateDir() { + createDir(testHttpFsFolder) + assertTrue("curl command to create a dir failed", sh.getRet() == 0); + assertValueExists(sh.getOut(), HTTPFS_SUCCESS); + } + + @Test + public void testRenameDir() { + createDir(testHttpFsFolder); + assertTrue("curl command to create a dir failed", sh.getRet() == 0); + assertValueExists(sh.getOut(), HTTPFS_SUCCESS); + sh.exec("curl -i -X PUT '$HTTPFS_PREFIX$testHttpFsFolder?user.name=$USERNAME&op=RENAME&destination=$testHttpFsFolderRenamed'"); + assertTrue("curl command to rename a dir failed", shHDFS.getRet() == 0); + assertValueExists(sh.getOut(), HTTPFS_SUCCESS); + } + + @Test + public void testDeleteDir() { + createDir(testHttpFsFolder); + assertTrue("curl command to create a dir failed", sh.getRet() == 0); + assertValueExists(sh.getOut(), HTTPFS_SUCCESS); + sh.exec("curl -i -X DELETE '$HTTPFS_PREFIX$testHttpFsFolder?user.name=$USERNAME&op=DELETE'"); + assertTrue("curl command to delete a dir failed", shHDFS.getRet() == 0); + assertValueExists(sh.getOut(), HTTPFS_SUCCESS); + } + + @Test + public void testStatusDir() { + createDir(testHttpFsFolder); + assertTrue("curl command to create a dir failed", sh.getRet() == 0); + assertValueExists(sh.getOut(), HTTPFS_SUCCESS); + sh.exec("curl -i '$HTTPFS_PREFIX$testHttpFsFolder?user.name=$USERNAME&op=GETFILESTATUS'"); + assertTrue("curl command to create a dir failed", sh.getRet() == 0); + assertValueExists(sh.getOut(), HTTPFS_SUCCESS); + assertValueExists(sh.getOut(), "DIRECTORY"); + } + + @Test + public void testCreateFile() { + String filename = "helloworld.txt"; + String filenameContent = 'Hello World!'; + + createDir(testHttpFsFolder); + assertTrue("curl command to create a dir failed", sh.getRet() == 0); + sh.exec("curl -i -X PUT '$HTTPFS_PREFIX$testHttpFsFolder/$filename?user.name=$USERNAME&op=CREATE'"); + assertTrue("curl command to create a file failed", shHDFS.getRet() == 0); + String datanodeLocation = null; + sh.getOut().each { + if (it.startsWith("Location:")) { + datanodeLocation = it.split(' ')[1]; + return true; + } + } + LOG.debug("Datanode location: $datanodeLocation"); + assertValueExists(sh.getOut(), HTTPFS_SUCCESS); + sh.exec("curl -i -T $DATA_DIR/$filename '$datanodeLocation' --header 'Content-Type:application/octet-stream'"); + assertTrue("curl command to create a file failed", shHDFS.getRet() == 0); + assertValueExists(sh.getOut(), HTTPFS_SUCCESS); + sh.exec("curl -i -L '$HTTPFS_PREFIX$testHttpFsFolder/$filename?user.name=$USERNAME&op=OPEN'"); + assertTrue("curl command to create a file failed", shHDFS.getRet() == 0); + assertValueExists(sh.getOut(), HTTPFS_SUCCESS); + assertValueExists(sh.getOut(), filenameContent); + } +} + http://git-wip-us.apache.org/repos/asf/bigtop/blob/69f7c942/bigtop-tests/test-artifacts/httpfs/src/main/resources/text-files/helloworld.txt ---------------------------------------------------------------------- diff --git a/bigtop-tests/test-artifacts/httpfs/src/main/resources/text-files/helloworld.txt b/bigtop-tests/test-artifacts/httpfs/src/main/resources/text-files/helloworld.txt new file mode 100644 index 0000000..980a0d5 --- /dev/null +++ b/bigtop-tests/test-artifacts/httpfs/src/main/resources/text-files/helloworld.txt @@ -0,0 +1 @@ +Hello World! http://git-wip-us.apache.org/repos/asf/bigtop/blob/69f7c942/bigtop-tests/test-artifacts/pom.xml ---------------------------------------------------------------------- diff --git a/bigtop-tests/test-artifacts/pom.xml b/bigtop-tests/test-artifacts/pom.xml index e3963cb..a7911e8 100644 --- a/bigtop-tests/test-artifacts/pom.xml +++ b/bigtop-tests/test-artifacts/pom.xml @@ -35,6 +35,7 @@ <modules> <module>hadoop</module> <module>package</module> + <module>httpfs</module> <module>oozie</module> <module>hive</module> <module>hbase</module> http://git-wip-us.apache.org/repos/asf/bigtop/blob/69f7c942/bigtop-tests/test-execution/smokes/httpfs/pom.xml ---------------------------------------------------------------------- diff --git a/bigtop-tests/test-execution/smokes/httpfs/pom.xml b/bigtop-tests/test-execution/smokes/httpfs/pom.xml new file mode 100644 index 0000000..74696e2 --- /dev/null +++ b/bigtop-tests/test-execution/smokes/httpfs/pom.xml @@ -0,0 +1,97 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<project xmlns="http://maven.apache.org/POM/4.0.0"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + + <parent> + <groupId>org.apache.bigtop.itest</groupId> + <artifactId>smoke-tests</artifactId> + <version>0.8.0-SNAPSHOT</version> + <relativePath>../pom.xml</relativePath> + </parent> + + <modelVersion>4.0.0</modelVersion> + <groupId>org.apache.bigtop.itest</groupId> + <artifactId>httpfs-smoke-execution</artifactId> + <version>0.8.0-SNAPSHOT</version> + <name>Httpfs smoke test execution</name> + + <properties> + <org.apache.maven-dependency-plugin.groupId>org.apache.bigtop.itest</org.apache.maven-dependency-plugin.groupId> + <org.apache.maven-dependency-plugin.artifactId>httpfs-smoke</org.apache.maven-dependency-plugin.artifactId> + <org.apache.maven-dependency-plugin.version>${httpfs-smoke.version}</org.apache.maven-dependency-plugin.version> + <org.apache.maven-dependency-plugin.type>jar</org.apache.maven-dependency-plugin.type> + + <HADOOP_HOME>${env.HADOOP_HOME}</HADOOP_HOME> + <HADOOP_CONF_DIR>${env.HADOOP_CONF_DIR}</HADOOP_CONF_DIR> + </properties> + + <dependencies> + <dependency> + <groupId>org.apache.bigtop.itest</groupId> + <artifactId>httpfs-smoke</artifactId> + </dependency> + </dependencies> + + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-enforcer-plugin</artifactId> + <version>1.0</version> + <executions> + <execution> + <id>enforce-property</id> + <goals> + <goal>enforce</goal> + </goals> + <configuration> + <rules> + <requireProperty> + <property>HADOOP_HOME</property> + <message>HADOOP_HOME env. variable has to be set</message> + </requireProperty> + <requireProperty> + <property>HADOOP_CONF_DIR</property> + <message>HADOOP_CONF_DIR env. variable has to be set</message> + </requireProperty> + </rules> + <fail>true</fail> + </configuration> + </execution> + </executions> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-dependency-plugin</artifactId> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-failsafe-plugin</artifactId> + <version>2.11</version> + <configuration> + <additionalClasspathElements> + <additionalClasspathElement>${HADOOP_CONF_DIR}</additionalClasspathElement> + </additionalClasspathElements> + </configuration> + </plugin> + </plugins> + </build> + +</project> http://git-wip-us.apache.org/repos/asf/bigtop/blob/69f7c942/bigtop-tests/test-execution/smokes/pom.xml ---------------------------------------------------------------------- diff --git a/bigtop-tests/test-execution/smokes/pom.xml b/bigtop-tests/test-execution/smokes/pom.xml index 7ccc8bd..1da31b2 100644 --- a/bigtop-tests/test-execution/smokes/pom.xml +++ b/bigtop-tests/test-execution/smokes/pom.xml @@ -37,6 +37,7 @@ <itest-conf.version>${project.version}</itest-conf.version> <itest-common.version>${project.version}</itest-common.version> <hadoop-smoke.version>${project.version}</hadoop-smoke.version> + <httpfs-smoke.version>${project.version}</httpfs-smoke.version> <flume-smoke.version>${project.version}</flume-smoke.version> <hbase-smoke.version>${project.version}</hbase-smoke.version> <hive-smoke.version>${project.version}</hive-smoke.version> @@ -64,8 +65,8 @@ <module>giraph</module> <module>hue</module> <module>crunch</module> - <!-- + <module>httpfs</module> <module>flume</module> <module>sqoop</module> --> @@ -154,6 +155,12 @@ <version>${phoenix-smoke.version}</version> <scope>test</scope> </dependency> + <dependency> + <groupId>org.apache.bigtop.itest</groupId> + <artifactId>httpfs-smoke</artifactId> + <version>${httpfs-smoke.version}</version> + <scope>test</scope> + </dependency> </dependencies> </dependencyManagement>
