Repository: bigtop Updated Branches: refs/heads/master 3bc16d0d0 -> 2d5fd34fd
BIGTOP-1334. Add DFS tests to TestCLI Signed-off-by: Konstantin Boudnik <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/2d5fd34f Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/2d5fd34f Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/2d5fd34f Branch: refs/heads/master Commit: 2d5fd34fddbf1d40a6aeb8aa5d39e2c6c630cff5 Parents: 3bc16d0 Author: Dasha Boudnik <[email protected]> Authored: Tue Jan 20 20:03:17 2015 -0800 Committer: Konstantin Boudnik <[email protected]> Committed: Wed Jan 21 15:52:10 2015 -0800 ---------------------------------------------------------------------- .../bigtop/itest/hadoop/hcfs/TestDFSCLI.java | 109 ++++++++ .../main/resources/clitest_data/testDFSConf.xml | 252 +++++++++++++++++++ 2 files changed, 361 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/bigtop/blob/2d5fd34f/bigtop-tests/test-artifacts/hadoop/src/main/groovy/org/apache/bigtop/itest/hadoop/hcfs/TestDFSCLI.java ---------------------------------------------------------------------- diff --git a/bigtop-tests/test-artifacts/hadoop/src/main/groovy/org/apache/bigtop/itest/hadoop/hcfs/TestDFSCLI.java b/bigtop-tests/test-artifacts/hadoop/src/main/groovy/org/apache/bigtop/itest/hadoop/hcfs/TestDFSCLI.java new file mode 100644 index 0000000..586c0e5 --- /dev/null +++ b/bigtop-tests/test-artifacts/hadoop/src/main/groovy/org/apache/bigtop/itest/hadoop/hcfs/TestDFSCLI.java @@ -0,0 +1,109 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.bigtop.itest.hadoop.hcfs; + +import org.apache.bigtop.itest.shell.Shell; +import org.apache.commons.lang.StringUtils; +import org.apache.hadoop.cli.TestHDFSCLI; +import org.apache.hadoop.fs.CommonConfigurationKeys; +import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.hdfs.HdfsConfiguration; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; + +import java.io.File; + +public class TestDFSCLI extends TestHDFSCLI { + public static final String TEST_DIR_ABSOLUTE = "/tmp/testcli_" + Long.valueOf(System.currentTimeMillis()); + public static String NAMENODE_TESTDIR_HACK; + private String supergroup; + private static Shell shHDFS = new Shell("/bin/bash"); + + @Before + @Override + public void setUp() throws Exception { + readTestConfigFile(); + + // Configuration of real Hadoop cluster + conf = new HdfsConfiguration(); + supergroup = System.getProperty("hcfs.root.groupname", + conf.get(DFSConfigKeys.DFS_PERMISSIONS_SUPERUSERGROUP_KEY)); + namenode = conf.get(CommonConfigurationKeys.FS_DEFAULT_NAME_KEY); + username = System.getProperty("user.name"); + + conf.setBoolean(CommonConfigurationKeys.HADOOP_SECURITY_AUTHORIZATION, true); + // Many of the tests expect a replication value of 1 in the output + conf.setInt("dfs.replication", 1); + + clitestDataDir = new File(TEST_CACHE_DATA_DIR).toURI().toString().replace(' ', '+'); + + String[] createTestcliDirCmds = { + "hadoop fs -mkdir -p " + TEST_DIR_ABSOLUTE, + "hadoop fs -chmod 777 " + TEST_DIR_ABSOLUTE + }; + shHDFS.exec(createTestcliDirCmds); + + // Check assumptions which would make some cases fail if not met + Assert.assertEquals("Creation of testcli dir should succeed and return 0" + + " (but it failed with the following error message: " + + StringUtils.join(shHDFS.getErr().toArray(), "\\n") + ")", + 0, shHDFS.getRet()); + // We can't just use conf.setInt(fs.trash.interval", 0) because if trash is + // enabled on the server, client configuration value is ignored. + Assert.assertEquals("HDFS trash should be disabled via fs.trash.interval", + 0, conf.getInt("fs.trash.interval", 0)); + Assert.assertEquals("This test needs to be run under root user of hcfs", + System.getProperty("hcfs.root.username", "hdfs"), username); + } + + @After + @Override + public void tearDown() throws Exception { + super.tearDown(); + + String removeTestcliDirCmd = "hadoop fs -rm -r " + TEST_DIR_ABSOLUTE; + shHDFS.exec(removeTestcliDirCmd); + } + + @Override + protected String getTestFile() { + return "testDFSConf.xml"; + } + + /** + * Expand commands from the test config file. + * This method is used in displayResults() and compareTestOutput() only, + * so it doesn't have any effect on the test execution itself. + * + * @param cmd executed command + * @return String expanded command + */ + @Override + protected String expandCommand(String cmd) { + String expCmd = super.expandCommand(cmd); + // note: super.expandCommand() expands CLITEST_DATA and USERNAME + expCmd = expCmd.replaceAll("NAMENODETEST_DIR_ABSOLUTE", NAMENODE_TESTDIR_HACK); + expCmd = expCmd.replaceAll("TEST_DIR_ABSOLUTE", TEST_DIR_ABSOLUTE); + expCmd = expCmd.replaceAll("supergroup", supergroup); + expCmd = expCmd.replaceAll("NAMENODE", namenode); + expCmd = expCmd.replaceAll("USER_NAME", System.getProperty("user.name")); + return expCmd; + } +} http://git-wip-us.apache.org/repos/asf/bigtop/blob/2d5fd34f/bigtop-tests/test-artifacts/hadoop/src/main/resources/clitest_data/testDFSConf.xml ---------------------------------------------------------------------- diff --git a/bigtop-tests/test-artifacts/hadoop/src/main/resources/clitest_data/testDFSConf.xml b/bigtop-tests/test-artifacts/hadoop/src/main/resources/clitest_data/testDFSConf.xml new file mode 100644 index 0000000..2f601f8 --- /dev/null +++ b/bigtop-tests/test-artifacts/hadoop/src/main/resources/clitest_data/testDFSConf.xml @@ -0,0 +1,252 @@ +<?xml version="1.0" encoding="UTF-8"?> +<?xml-stylesheet type="text/xsl" href="testConf.xsl"?> + +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<configuration> + <!-- Normal mode is test. To run just the commands and dump the output + to the log, set it to nocompare --> + <mode>test</mode> + +<tests> + + <test> + <description>allowSnapshot</description> + <test-commands> + <command>-fs NAMENODE -mkdir -p /user/USERNAME/dir1</command> + <dfs-admin-command>-fs NAMENODE -allowSnapshot /user/USERNAME/dir1</dfs-admin-command> + </test-commands> + <cleanup-commands> + <dfs-admin-command>-fs NAMENODE -disallowSnapshot /user/USERNAME/dir1</dfs-admin-command> + <command>-fs NAMENODE -rm -r /user/USERNAME</command> + </cleanup-commands> + <comparators> + <comparator> + <type>RegexpComparator</type> + <expected-output>Allowing snaphot on /user/USERNAME/dir1 succeeded</expected-output> + </comparator> + </comparators> + </test> + + <test> + <description>disallowSnapshot</description> + <test-commands> + <command>-fs NAMENODE -mkdir -p /user/USERNAME/dir1</command> + <dfs-admin-command>-fs NAMENODE -allowSnapshot /user/USERNAME/dir1</dfs-admin-command> + <dfs-admin-command>-fs NAMENODE -disallowSnapshot /user/USERNAME/dir1</dfs-admin-command> + </test-commands> + <cleanup-commands> + <command>-fs NAMENODE -rm -r /user/USERNAME</command> + </cleanup-commands> + <comparators> + <comparator> + <type>RegexpComparator</type> + <expected-output>Disallowing snaphot on /user/USERNAME/dir1 succeeded</expected-output> + </comparator> + </comparators> + </test> + + <test> + <description>createSnapshot</description> + <test-commands> + <command>-fs NAMENODE -mkdir -p /user/USERNAME/dir1</command> + <dfs-admin-command>-fs NAMENODE -allowSnapshot /user/USERNAME/dir1</dfs-admin-command> + <command>-fs NAMENODE -createSnapshot /user/USERNAME/dir1 snapshot1</command> + </test-commands> + <cleanup-commands> + <command>-fs NAMENODE -deleteSnapshot /user/USERNAME/dir1 snapshot1</command> + <dfs-admin-command>-fs NAMENODE -disallowSnapshot /user/USERNAME/dir1</dfs-admin-command> + <command>-fs NAMENODE -rm -r /user/USERNAME</command> + </cleanup-commands> + <comparators> + <comparator> + <type>RegexpComparator</type> + <expected-output>Created snapshot /user/USERNAME/dir1/.snapshot/snapshot1</expected-output> + </comparator> + </comparators> + </test> + + <test> + <description>renameSnapshot</description> + <test-commands> + <command>-fs NAMENODE -mkdir -p /user/USERNAME/dir1</command> + <dfs-admin-command>-fs NAMENODE -allowSnapshot /user/USERNAME/dir1</dfs-admin-command> + <command>-fs NAMENODE -createSnapshot /user/USERNAME/dir1 snapshot1</command> + <command>-fs NAMENODE -renameSnapshot /user/USERNAME/dir1 snapshot1 snapshot2</command> + <command>-fs NAMENODE -ls /user/USERNAME/dir1/.snapshot</command> + </test-commands> + <cleanup-commands> + <command>-fs NAMENODE -deleteSnapshot /user/USERNAME/dir1 snapshot2</command> + <dfs-admin-command>-fs NAMENODE -disallowSnapshot /user/USERNAME/dir1</dfs-admin-command> + <command>-fs NAMENODE -rm -r /user/USERNAME</command> + </cleanup-commands> + <comparators> + <comparator> + <type>RegexpComparator</type> + <expected-output>Found 1 items</expected-output> + </comparator> + <comparator> + <type>RegexpComparator</type> + <expected-output>^drwxr-xr-x( )*-( )*USERNAME( )*supergroup( )*0( )*[0-9]{4,}-[0-9]{2,}-[0-9]{2,} [0-9]{2,}:[0-9]{2,}( )*/user/USERNAME/dir1/.snapshot/snapshot2</expected-output> + </comparator> + </comparators> + </test> + + <test> + <description>deleteSnapshot</description> + <test-commands> + <command>-fs NAMENODE -mkdir -p /user/USERNAME/dir1</command> + <dfs-admin-command>-fs NAMENODE -allowSnapshot /user/USERNAME/dir1</dfs-admin-command> + <command>-fs NAMENODE -createSnapshot /user/USERNAME/dir1 snapshot1</command> + <command>-fs NAMENODE -deleteSnapshot /user/USERNAME/dir1 snapshot1</command> + <command>-fs NAMENODE -ls /user/USERNAME/dir1/.snapshot</command> + </test-commands> + <cleanup-commands> + <dfs-admin-command>-fs NAMENODE -disallowSnapshot /user/USERNAME/dir1</dfs-admin-command> + <command>-fs NAMENODE -rm -r /user/USERNAME</command> + </cleanup-commands> + <comparators> + <comparator> + <type>ExactComparator</type> + <expected-output></expected-output> + </comparator> + </comparators> + </test> + + <!-- DFSadmin tests --> + <test> + <description>refreshUserToGroupsMappings</description> + <test-commands> + <dfs-admin-command>-fs NAMENODE -refreshUserToGroupsMappings</dfs-admin-command> + </test-commands> + <cleanup-commands> + <command></command> + </cleanup-commands> + <comparators> + <comparator> + <type>SubstringComparator</type> + <expected-output></expected-output> + </comparator> + </comparators> + </test> + + <test> + <description>refreshSuperUserGroupsConfiguration</description> + <test-commands> + <dfs-admin-command>-fs NAMENODE -refreshSuperUserGroupsConfiguration</dfs-admin-command> + </test-commands> + <cleanup-commands> + <command></command> + </cleanup-commands> + <comparators> + <comparator> + <type>SubstringComparator</type> + <expected-output></expected-output> + </comparator> + </comparators> + </test> + + <test> + <description>setQuota</description> + <test-commands> + <command>-fs NAMENODE -mkdir -p /user/USERNAME/dir1</command> + <dfs-admin-command>-fs NAMENODE -setQuota 3 /user/USERNAME/dir1</dfs-admin-command> + <command>-fs NAMENODE -count -q /user/USERNAME/dir1</command> + </test-commands> + <cleanup-commands> + <command>-fs NAMENODE -rm -r /user/USERNAME</command> + </cleanup-commands> + <comparators> + <comparator> + <type>RegexpComparator</type> + <expected-output>( )*3.*/user/USERNAME/dir1</expected-output> + </comparator> + </comparators> + </test> + + <test> + <description>clrQuota</description> + <test-commands> + <command>-fs NAMENODE -mkdir -p /user/USERNAME/dir1</command> + <dfs-admin-command>-fs NAMENODE -setQuota 3 /user/USERNAME/dir1</dfs-admin-command> + <dfs-admin-command>-fs NAMENODE -clrQuota /user/USERNAME/dir1</dfs-admin-command> + <command>-fs NAMENODE -count -q /user/USERNAME/dir1</command> + </test-commands> + <cleanup-commands> + <command>-fs NAMENODE -rm -r /user/USERNAME</command> + </cleanup-commands> + <comparators> + <comparator> + <type>RegexpComparator</type> + <expected-output>( )*none.*/user/USERNAME/dir1</expected-output> + </comparator> + </comparators> + </test> + + <test> + <description>setSpaceQuota</description> + <test-commands> + <command>-fs NAMENODE -mkdir -p /user/USERNAME/dir1</command> + <dfs-admin-command>-fs NAMENODE -setSpaceQuota 1G /user/USERNAME/dir1</dfs-admin-command> + </test-commands> + <cleanup-commands> + <command>-fs NAMENODE -rm -r /user/USERNAME/dir1</command> + </cleanup-commands> + <comparators> + <comparator> + <type>ExactComparator</type> + <expected-output></expected-output> + </comparator> + </comparators> + </test> + + <test> + <description>setBalancerBandwidth</description> + <test-commands> + <dfs-admin-command>-fs NAMENODE -setBalancerBandwidth 104857600</dfs-admin-command> + </test-commands> + <cleanup-commands> + <command></command> + </cleanup-commands> + <comparators> + <comparator> + <type>SubstringComparator</type> + <expected-output></expected-output> + </comparator> + </comparators> + </test> + + <test> + <description>finalizeUpgrade</description> + <test-commands> + <dfs-admin-command>-fs NAMENODE -finalizeUpgrade</dfs-admin-command> + </test-commands> + <cleanup-commands> + <command></command> + </cleanup-commands> + <comparators> + <comparator> + <type>SubstringComparator</type> + <expected-output></expected-output> + </comparator> + </comparators> + </test> + +</tests> + +</configuration>
