Repository: mesos Updated Branches: refs/heads/master 925e99ea7 -> 293639961
Added tests for HDFS client. Review: https://reviews.apache.org/r/41384 Project: http://git-wip-us.apache.org/repos/asf/mesos/repo Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/29363996 Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/29363996 Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/29363996 Branch: refs/heads/master Commit: 293639961525e49e17dccd8d380684b8d668840d Parents: 00f6693 Author: Jie Yu <[email protected]> Authored: Mon Dec 14 17:19:31 2015 -0800 Committer: Jie Yu <[email protected]> Committed: Tue Dec 15 10:55:33 2015 -0800 ---------------------------------------------------------------------- src/Makefile.am | 1 + src/tests/hdfs_tests.cpp | 210 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 211 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/mesos/blob/29363996/src/Makefile.am ---------------------------------------------------------------------- diff --git a/src/Makefile.am b/src/Makefile.am index acd17de..8f6b98b 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -1688,6 +1688,7 @@ mesos_tests_SOURCES = \ tests/files_tests.cpp \ tests/flags.cpp \ tests/gc_tests.cpp \ + tests/hdfs_tests.cpp \ tests/health_check_tests.cpp \ tests/hierarchical_allocator_tests.cpp \ tests/hook_tests.cpp \ http://git-wip-us.apache.org/repos/asf/mesos/blob/29363996/src/tests/hdfs_tests.cpp ---------------------------------------------------------------------- diff --git a/src/tests/hdfs_tests.cpp b/src/tests/hdfs_tests.cpp new file mode 100644 index 0000000..29f1560 --- /dev/null +++ b/src/tests/hdfs_tests.cpp @@ -0,0 +1,210 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include <process/gtest.hpp> +#include <process/owned.hpp> + +#include <stout/check.hpp> +#include <stout/os.hpp> +#include <stout/path.hpp> + +#include <stout/os/touch.hpp> +#include <stout/os/write.hpp> + +#include <stout/tests/utils.hpp> + +#include "hdfs/hdfs.hpp" + +using std::string; + +using process::Future; +using process::Owned; + +namespace mesos { +namespace internal { +namespace tests { + +class HdfsTest : public TemporaryDirectoryTest +{ +public: + virtual void SetUp() + { + TemporaryDirectoryTest::SetUp(); + + // Create a fake hadoop command line tool. The tests serialize + // bash scripts into this file which emulates the hadoop client's + // logic while operating on the local filesystem. + hadoop = path::join(os::getcwd(), "hadoop"); + + ASSERT_SOME(os::touch(hadoop)); + + // Make sure the script has execution permission. + ASSERT_SOME(os::chmod( + hadoop, + S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH | S_IXOTH)); + } + +protected: + string hadoop; +}; + + +// This test verifies the 'HDFS::exists(path)' method. We emulate the +// hadoop client by testing the existence of a local file. +TEST_F(HdfsTest, Exists) +{ + // The script emulating 'hadoop fs -test -e <path>'. + // NOTE: We emulate a version call here which is exercised when + // creating the HDFS client. + ASSERT_SOME(os::write( + hadoop, + "#!/bin/sh\n" + "if [ \"$1\" = \"version\" ]; then\n" + " exit 0\n" + "fi\n" + "test -e $4\n")); + + Try<Owned<HDFS>> hdfs = HDFS::create(hadoop); + ASSERT_SOME(hdfs); + + Future<bool> exists = hdfs.get()->exists(hadoop); + AWAIT_READY(exists); + EXPECT_TRUE(exists.get()); + + exists = hdfs.get()->exists(path::join(os::getcwd(), "NotExists")); + AWAIT_READY(exists); + EXPECT_FALSE(exists.get()); +} + + +// This test verifies the 'HDFS::du(path)' method. We emulate the +// hadoop client by doing a 'du' on the local filesystem. +TEST_F(HdfsTest, Du) +{ + // The script emulating 'hadoop fs -du <path>'. + // NOTE: We emulate a version call here which is exercised when + // creating the HDFS client. + ASSERT_SOME(os::write( + hadoop, + "#!/bin/sh\n" + "if [ \"$1\" = \"version\" ]; then\n" + " exit 0\n" + "fi\n" + "du $3\n")); + + Try<Owned<HDFS>> hdfs = HDFS::create(hadoop); + ASSERT_SOME(hdfs); + + Future<Bytes> bytes = hdfs.get()->du(hadoop); + AWAIT_READY(bytes); + + bytes = hdfs.get()->du(path::join(os::getcwd(), "Invalid")); + AWAIT_FAILED(bytes); +} + + +// This test verifies the 'HDFS::rm(path)' method. We emulate the +// hadoop client by removing a file on the local filesystem. +TEST_F(HdfsTest, Rm) +{ + // The script emulating 'hadoop fs -rm <path>'. + // NOTE: We emulate a version call here which is exercised when + // creating the HDFS client. + ASSERT_SOME(os::write( + hadoop, + "#!/bin/sh\n" + "if [ \"$1\" = \"version\" ]; then\n" + " exit 0\n" + "fi\n" + "rm $3\n")); + + Try<Owned<HDFS>> hdfs = HDFS::create(hadoop); + ASSERT_SOME(hdfs); + + string file = path::join(os::getcwd(), "file"); + + ASSERT_SOME(os::touch(file)); + + Future<Nothing> rm = hdfs.get()->rm(file); + AWAIT_READY(rm); + + rm = hdfs.get()->rm(path::join(os::getcwd(), "Invalid")); + AWAIT_FAILED(rm); +} + + +// This test verifies the 'HDFS::copyFromLocal(from, to)' method. We +// emulate the hadoop client by doing a 'cp' on the local filesystem. +TEST_F(HdfsTest, CopyFromLocal) +{ + // The script emulating 'hadoop fs -copyFromLocal <from> <to>'. + // NOTE: We emulate a version call here which is exercised when + // creating the HDFS client. + ASSERT_SOME(os::write( + hadoop, + "#!/bin/sh\n" + "if [ \"$1\" = \"version\" ]; then\n" + " exit 0\n" + "fi\n" + "cp $3 $4")); + + Try<Owned<HDFS>> hdfs = HDFS::create(hadoop); + ASSERT_SOME(hdfs); + + string file1 = path::join(os::getcwd(), "file1"); + string file2 = path::join(os::getcwd(), "file2"); + + ASSERT_SOME(os::write(file1, "abc")); + + Future<Nothing> copy = hdfs.get()->copyFromLocal(file1, file2); + AWAIT_READY(copy); + + EXPECT_SOME_EQ("abc", os::read(file2)); +} + + +// This test verifies the 'HDFS::copyToLocal(from, to)' method. We +// emulate the hadoop client by doing a 'cp' on the local filesystem. +TEST_F(HdfsTest, CopyToLocal) +{ + // The script emulating 'hadoop fs -copyToLocal <from> <to>'. + // NOTE: We emulate a version call here which is exercised when + // creating the HDFS client. + ASSERT_SOME(os::write( + hadoop, + "#!/bin/sh\n" + "if [ \"$1\" = \"version\" ]; then\n" + " exit 0\n" + "fi\n" + "cp $3 $4")); + + Try<Owned<HDFS>> hdfs = HDFS::create(hadoop); + ASSERT_SOME(hdfs); + + string file1 = path::join(os::getcwd(), "file1"); + string file2 = path::join(os::getcwd(), "file2"); + + ASSERT_SOME(os::write(file1, "abc")); + + Future<Nothing> copy = hdfs.get()->copyToLocal(file1, file2); + AWAIT_READY(copy); + + EXPECT_SOME_EQ("abc", os::read(file2)); +} + +} // namespace tests { +} // namespace internal { +} // namespace mesos {
