IMPALA-4673: Use --local_library_dir for tzdb startup scratch space Currently we hardcode the the path to /tmp which may not be accessible on certain environments.
Change-Id: I7ab0238e6ce65ca39c167349e3b79aa70fb39d2f Reviewed-on: http://gerrit.cloudera.org:8080/5523 Reviewed-by: Bharath Vissapragada <[email protected]> Tested-by: Impala Public Jenkins Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/77dbdc3f Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/77dbdc3f Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/77dbdc3f Branch: refs/heads/master Commit: 77dbdc3f572260a90af174dcb9112cb991ec296c Parents: 8e45c4a Author: Bharath Vissapragada <[email protected]> Authored: Thu Dec 15 12:03:16 2016 -0800 Committer: Impala Public Jenkins <[email protected]> Committed: Mon Mar 27 21:53:04 2017 +0000 ---------------------------------------------------------------------- be/src/common/init.cc | 6 ++++-- be/src/exprs/expr-test.cc | 2 ++ be/src/exprs/timezone_db.cc | 15 ++++++++++----- be/src/exprs/timezone_db.h | 2 +- be/src/runtime/lib-cache.cc | 3 +-- be/src/service/impalad-main.cc | 2 ++ 6 files changed, 20 insertions(+), 10 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/77dbdc3f/be/src/common/init.cc ---------------------------------------------------------------------- diff --git a/be/src/common/init.cc b/be/src/common/init.cc index 1e90f53..16f0f41 100644 --- a/be/src/common/init.cc +++ b/be/src/common/init.cc @@ -24,7 +24,6 @@ #include "common/status.h" #include "exec/kudu-util.h" #include "exprs/expr.h" -#include "exprs/timezone_db.h" #include "gutil/atomicops.h" #include "rpc/authentication.h" #include "rpc/thrift-util.h" @@ -76,6 +75,10 @@ DEFINE_int64(pause_monitor_warn_threshold_ms, 10000, "If the pause monitor sleep "more than this time period, a warning is logged. If set to 0 or less, pause monitor" " is disabled."); +DEFINE_string(local_library_dir, "/tmp", + "Scratch space for local fs operations. Currently used for copying " + "UDF binaries locally from HDFS and also for initializing the timezone db"); + // Defined by glog. This allows users to specify the log level using a glob. For // example -vmodule=*scanner*=3 would enable full logging for scanners. If redaction // is enabled, this option won't be allowed because some logging dumps table data @@ -198,7 +201,6 @@ void impala::InitCommonRuntime(int argc, char** argv, bool init_jvm, impala::InitThreading(); impala::TimestampParser::Init(); impala::SeedOpenSSLRNG(); - ABORT_IF_ERROR(impala::TimezoneDatabase::Initialize()); ABORT_IF_ERROR(impala::InitAuth(argv[0])); // Initialize maintenance_thread after InitGoogleLoggingSafe and InitThreading. http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/77dbdc3f/be/src/exprs/expr-test.cc ---------------------------------------------------------------------- diff --git a/be/src/exprs/expr-test.cc b/be/src/exprs/expr-test.cc index 0ca2da2..0de6833 100644 --- a/be/src/exprs/expr-test.cc +++ b/be/src/exprs/expr-test.cc @@ -38,6 +38,7 @@ #include "exprs/null-literal.h" #include "exprs/string-functions.h" #include "exprs/timestamp-functions.h" +#include "exprs/timezone_db.h" #include "gen-cpp/Exprs_types.h" #include "gen-cpp/hive_metastore_types.h" #include "rpc/thrift-client.h" @@ -7072,6 +7073,7 @@ TEST_F(ExprTest, UuidTest) { int main(int argc, char **argv) { ::testing::InitGoogleTest(&argc, argv); InitCommonRuntime(argc, argv, true, TestInfo::BE_TEST); + ABORT_IF_ERROR(TimezoneDatabase::Initialize()); InitFeSupport(false); impala::LlvmCodeGen::InitializeLlvm(); http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/77dbdc3f/be/src/exprs/timezone_db.cc ---------------------------------------------------------------------- diff --git a/be/src/exprs/timezone_db.cc b/be/src/exprs/timezone_db.cc index c7ae888..110d23b 100644 --- a/be/src/exprs/timezone_db.cc +++ b/be/src/exprs/timezone_db.cc @@ -19,6 +19,7 @@ #include <boost/date_time/compiler_config.hpp> #include <boost/date_time/local_time/posix_time_zone.hpp> +#include <boost/filesystem.hpp> #include "exprs/timestamp-functions.h" #include "gutil/strings/substitute.h" @@ -29,6 +30,8 @@ using boost::local_time::tz_database; using boost::local_time::time_zone_ptr; using boost::local_time::posix_time_zone; +DECLARE_string(local_library_dir); + namespace impala { tz_database TimezoneDatabase::tz_database_; @@ -666,14 +669,16 @@ const char* TimezoneDatabase::TIMEZONE_DATABASE_STR = "\"ID\",\"STD ABBR\",\"STD Status TimezoneDatabase::Initialize() { // Create a temporary file and write the timezone information. The boost - // interface only loads this format from a file. We don't want to raise - // an error here since this is done when the backend is created and this - // information might not actually get used by any queries. - char filestr[] = "/tmp/impala.tzdb.XXXXXXX"; + // interface only loads this format from a file. We abort the startup if + // this initialization fails for some reason. + char *filestr = const_cast<char*>((boost::filesystem::path(FLAGS_local_library_dir) + / string("impala.tzdb.XXXXXXX")).string().c_str()); FILE* file; int fd; if ((fd = mkstemp(filestr)) == -1) { - return Status(Substitute("Could not create temporary timezone file: $0", filestr)); + return Status(Substitute("Could not create temporary timezone file: $0. Check that " + "the directory $1 is writable by the user running Impala.", filestr, + FLAGS_local_library_dir)); } if ((file = fopen(filestr, "w")) == NULL) { unlink(filestr); http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/77dbdc3f/be/src/exprs/timezone_db.h ---------------------------------------------------------------------- diff --git a/be/src/exprs/timezone_db.h b/be/src/exprs/timezone_db.h index 20ea205..3a1178a 100644 --- a/be/src/exprs/timezone_db.h +++ b/be/src/exprs/timezone_db.h @@ -32,7 +32,7 @@ namespace impala { class TimezoneDatabase { public: /// Set up the static timezone database. - static Status Initialize(); + static Status Initialize() WARN_UNUSED_RESULT; /// Converts the name of a timezone to a boost timezone object. In some cases, the /// timestamp is required to determine the timezone because occasionally timezone http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/77dbdc3f/be/src/runtime/lib-cache.cc ---------------------------------------------------------------------- diff --git a/be/src/runtime/lib-cache.cc b/be/src/runtime/lib-cache.cc index 9a8d873..d694c49 100644 --- a/be/src/runtime/lib-cache.cc +++ b/be/src/runtime/lib-cache.cc @@ -33,8 +33,7 @@ namespace filesystem = boost::filesystem; -DEFINE_string(local_library_dir, "/tmp", - "Local directory to copy UDF libraries from HDFS into"); +DECLARE_string(local_library_dir); namespace impala { http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/77dbdc3f/be/src/service/impalad-main.cc ---------------------------------------------------------------------- diff --git a/be/src/service/impalad-main.cc b/be/src/service/impalad-main.cc index 341399c..4848b45 100644 --- a/be/src/service/impalad-main.cc +++ b/be/src/service/impalad-main.cc @@ -27,6 +27,7 @@ #include "exec/hbase-table-scanner.h" #include "exec/hbase-table-writer.h" #include "exprs/hive-udf-call.h" +#include "exprs/timezone_db.h" #include "runtime/hbase-table.h" #include "codegen/llvm-codegen.h" #include "common/status.h" @@ -59,6 +60,7 @@ DECLARE_bool(enable_rm); int ImpaladMain(int argc, char** argv) { InitCommonRuntime(argc, argv, true); + ABORT_IF_ERROR(TimezoneDatabase::Initialize()); ABORT_IF_ERROR(LlvmCodeGen::InitializeLlvm()); JniUtil::InitLibhdfs(); ABORT_IF_ERROR(HBaseTableScanner::Init());
