IMPALA-6249: Expose several build flags via web UI
Exposes a list of build flags via the impalad web UI. The build flags
can be viewed on the root page under the "Version" section. They can
be accessed via other tests through the debug version of the root page
(e.g. adding &json to the URL). The build flags are listed in a JSON
array so that they can be parsed easily. This should help run Impala
tests against a remote Impala cluster.
The build flags are read in CMakeLists.txt and then stored in
preprocessor variables.
Three build flags are exposed as part of this commit:
- Is_NDEBUG = [true, false]
- Whether NDEBUG was true or false at compile time
- CMake_Build_Type = [DEBUG, RELEASE, ADDRESS_SANITIZER, TIDY, UBSAN,
UBSAN_FULL, TSAN, CODE_COVERAGE_RELEASE, CODE_COVERAGE_DEBUG]
- The value of CMAKE_BUILD_TYPE at compile time
- Library_Link_Type = [DYNAMIC, STATIC]
- Derived from the compile time value of BUILD_SHARED_LIBS
There are a few other minor changes that are apart of this commit:
* The patch modifies environ.py so that it supports fetching build metadata
for both local and remote clusters.
* The tests under the tests/webserver directory were not being run because
'webserver' was not whitelisted in tests/run-tests.py. This patch fixes
that and addresses several test failures in run-tests.py.
* It reverts part of IMPALA-6947 so that their is no dependency from
start-impala-cluster.py to environ.py. The timeout discussed IMPALA-6947
is now set at compile time.
Testing:
Added new tests to webserver/test_web_pages.py to ensure that the build
flags are being set. Some tests are only run when run against a local
cluster because we have no way of getting the build info from a remote
cluster, whereas local clusters contain a .cmake_build_type file.
Change-Id: I47e3ad4cbf844909bdaf22a6f9d7bd915dce3f19
Reviewed-on: http://gerrit.cloudera.org:8080/11410
Reviewed-by: Impala Public Jenkins <impala-public-jenk...@cloudera.com>
Tested-by: Impala Public Jenkins <impala-public-jenk...@cloudera.com>
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/9bdb73b1
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/9bdb73b1
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/9bdb73b1
Branch: refs/heads/branch-3.1.0
Commit: 9bdb73b188c4c87ee3a71e56aa10de0eba32b086
Parents: c73530d
Author: Sahil Takiar <takiar.sa...@gmail.com>
Authored: Wed Sep 5 10:52:49 2018 -0400
Committer: Zoltan Borok-Nagy <borokna...@cloudera.com>
Committed: Tue Nov 13 12:50:23 2018 +0100
----------------------------------------------------------------------
CMakeLists.txt | 20 +-
be/src/common/config.h.in | 3 +
be/src/common/global-flags.cc | 16 +-
be/src/util/debug-util.h | 19 ++
be/src/util/default-path-handlers.cc | 27 +++
bin/start-impala-cluster.py | 4 +-
tests/common/environ.py | 207 +++++++++++++++----
tests/common/skip.py | 8 +-
tests/conftest.py | 4 +-
.../custom_cluster/test_admission_controller.py | 6 +-
.../test_automatic_invalidation.py | 5 +-
tests/custom_cluster/test_exchange_delays.py | 4 +-
tests/custom_cluster/test_metadata_replicas.py | 1 -
tests/custom_cluster/test_restart_services.py | 4 +-
tests/query_test/test_hdfs_caching.py | 4 +-
tests/query_test/test_runtime_filters.py | 4 +-
tests/run-tests.py | 19 +-
tests/statestore/test_statestore.py | 8 +-
tests/webserver/test_web_pages.py | 96 +++++++--
www/root.tmpl | 5 +-
20 files changed, 368 insertions(+), 96 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/impala/blob/9bdb73b1/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 90b4df8..e248540 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -30,6 +30,9 @@ if (NOT DEFINED BUILD_SHARED_LIBS)
set(BUILD_SHARED_LIBS OFF)
endif()
+# Store BUILD_SHARED_LIBS in a variable so it can be read in config.h.in
+set(IMPALA_BUILD_SHARED_LIBS ${BUILD_SHARED_LIBS})
+
# Build compile commands database
set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
@@ -47,8 +50,14 @@ endif(NOT CMAKE_BUILD_TYPE)
STRING (TOUPPER ${CMAKE_BUILD_TYPE} CMAKE_BUILD_TYPE)
message(STATUS "Build type is ${CMAKE_BUILD_TYPE}")
-# Write the build type to a file so that tests can determine the current build
type.
-file(WRITE "${CMAKE_SOURCE_DIR}/.cmake_build_type" ${CMAKE_BUILD_TYPE})
+
+# Write build flags to a file so that they can be read by tests
+file(WRITE "${CMAKE_SOURCE_DIR}/.cmake_build_type" ${CMAKE_BUILD_TYPE}\n)
+file(APPEND "${CMAKE_SOURCE_DIR}/.cmake_build_type" ${BUILD_SHARED_LIBS}\n)
+
+# Store CMAKE_BUILD_TYPE in a variable so it can be read in config.h.in
+string(REPLACE "_" "-" ESCAPED_CMAKE_BUILD_TYPE ${CMAKE_BUILD_TYPE})
+set(IMPALA_CMAKE_BUILD_TYPE ${ESCAPED_CMAKE_BUILD_TYPE})
set(ENABLE_CODE_COVERAGE false)
if ("${CMAKE_BUILD_TYPE}" STREQUAL "CODE_COVERAGE_DEBUG")
@@ -61,6 +70,13 @@ endif()
message(STATUS "ENABLE_CODE_COVERAGE: ${ENABLE_CODE_COVERAGE}")
+if (ENABLE_CODE_COVERAGE
+ OR "${CMAKE_BUILD_TYPE}" STREQUAL "ADDRESS_SANITIZER"
+ OR "${CMAKE_BUILD_TYPE}" STREQUAL "TSAN"
+ OR "${CMAKE_BUILD_TYPE}" STREQUAL "UBSAN")
+ set (SLOW_BUILD true)
+endif()
+
# Helper function that given a package name constructs the package_ROOT
variable based on
# the version number extracted from the environment
function(set_dep_root NAME)
http://git-wip-us.apache.org/repos/asf/impala/blob/9bdb73b1/be/src/common/config.h.in
----------------------------------------------------------------------
diff --git a/be/src/common/config.h.in b/be/src/common/config.h.in
index f474a9f..43c7e65 100644
--- a/be/src/common/config.h.in
+++ b/be/src/common/config.h.in
@@ -28,5 +28,8 @@
#cmakedefine HAVE_PIPE2
#cmakedefine HAVE_MAGIC_H
#cmakedefine HAVE_SYNC_FILE_RANGE
+#cmakedefine SLOW_BUILD
+#cmakedefine IMPALA_BUILD_SHARED_LIBS @IMPALA_BUILD_SHARED_LIBS@
+#cmakedefine IMPALA_CMAKE_BUILD_TYPE @IMPALA_CMAKE_BUILD_TYPE@
#endif
http://git-wip-us.apache.org/repos/asf/impala/blob/9bdb73b1/be/src/common/global-flags.cc
----------------------------------------------------------------------
diff --git a/be/src/common/global-flags.cc b/be/src/common/global-flags.cc
index 0473352..3c73b56 100644
--- a/be/src/common/global-flags.cc
+++ b/be/src/common/global-flags.cc
@@ -161,12 +161,18 @@ DEFINE_int32(kudu_operation_timeout_ms, 3 * 60 * 1000,
"Timeout (milliseconds) s
"all Kudu operations. This must be a positive value, and there is no way
to disable "
"timeouts.");
+#ifdef SLOW_BUILD
+static const int32 default_kudu_client_rpc_timeout_ms = 60000;
+#else
+static const int32 default_kudu_client_rpc_timeout_ms = 0;
+#endif
+
// Timeout (ms) for Kudu rpcs set in the BE on the KuduClient.
-DEFINE_int32(kudu_client_rpc_timeout_ms, 0, "(Advanced) Timeout (milliseconds)
set for "
- "individual Kudu client rpcs. An operation may consist of several rpcs, so
this is "
- "expected to be less than kudu_operation_timeout_ms. This must be a
positive value "
- "or it will be ignored and Kudu's default of 10s will be used. There is no
way to "
- "disable timeouts.");
+DEFINE_int32(kudu_client_rpc_timeout_ms, default_kudu_client_rpc_timeout_ms,
+ "(Advanced) Timeout (milliseconds) set for individual Kudu client rpcs. An
operation "
+ "may consist of several rpcs, so this is expected to be less than "
+ "kudu_operation_timeout_ms. This must be a positive value or it will be
ignored and "
+ "Kudu's default of 10s will be used. There is no way to disable
timeouts.");
DEFINE_int64(inc_stats_size_limit_bytes, 200 * (1LL<<20), "Maximum size of "
"incremental stats the catalog is allowed to serialize per table. "
http://git-wip-us.apache.org/repos/asf/impala/blob/9bdb73b1/be/src/util/debug-util.h
----------------------------------------------------------------------
diff --git a/be/src/util/debug-util.h b/be/src/util/debug-util.h
index 005cc23..240eab6 100644
--- a/be/src/util/debug-util.h
+++ b/be/src/util/debug-util.h
@@ -24,6 +24,7 @@
#include <thrift/protocol/TDebugProtocol.h>
+#include "common/config.h"
#include "gen-cpp/JniCatalog_types.h"
#include "gen-cpp/Descriptors_types.h"
#include "gen-cpp/Exprs_types.h"
@@ -97,6 +98,24 @@ bool ParseId(const std::string& s, TUniqueId* id);
/// This is used to set gflags build version
std::string GetBuildVersion(bool compact = false);
+#ifndef IMPALA_CMAKE_BUILD_TYPE
+static_assert(false, "IMPALA_CMAKE_BUILD_TYPE is not defined");
+#endif
+
+/// Returns the value of CMAKE_BUILD_TYPE used to build the code
+constexpr const char* GetCMakeBuildType() {
+ return AS_STRING(IMPALA_CMAKE_BUILD_TYPE);
+}
+
+/// Returns whether the code was dynamically or statically linked, return
+/// value is either STATIC or DYNAMIC.
+constexpr const char* GetLibraryLinkType() {
+ return AS_STRING(IMPALA_BUILD_SHARED_LIBS)[0] == 'O'
+ && AS_STRING(IMPALA_BUILD_SHARED_LIBS)[1] == 'N' ?
+ "DYNAMIC" :
+ "STATIC";
+}
+
/// Returns "<program short name> version <GetBuildVersion(compact)>"
std::string GetVersionString(bool compact = false);
http://git-wip-us.apache.org/repos/asf/impala/blob/9bdb73b1/be/src/util/default-path-handlers.cc
----------------------------------------------------------------------
diff --git a/be/src/util/default-path-handlers.cc
b/be/src/util/default-path-handlers.cc
index c492d06..df04d92 100644
--- a/be/src/util/default-path-handlers.cc
+++ b/be/src/util/default-path-handlers.cc
@@ -224,11 +224,38 @@ void JmxHandler(const Webserver::ArgumentMap& args,
Document* document) {
}
}
+// Helper function that creates a Value for a given build flag name, value and
adds it to
+// an array of build_flags
+void AddBuildFlag(const std::string& flag_name, const std::string& flag_value,
+ Document* document, Value* build_flags) {
+ Value build_type(kObjectType);
+ Value build_type_name(flag_name.c_str(), document->GetAllocator());
+ build_type.AddMember("flag_name", build_type_name, document->GetAllocator());
+ Value build_type_value(flag_value.c_str(), document->GetAllocator());
+ build_type.AddMember("flag_value", build_type_value,
document->GetAllocator());
+ build_flags->PushBack(build_type, document->GetAllocator());
+}
+
namespace impala {
void RootHandler(const Webserver::ArgumentMap& args, Document* document) {
Value version(GetVersionString().c_str(), document->GetAllocator());
document->AddMember("version", version, document->GetAllocator());
+
+#ifdef NDEBUG
+ const char* is_ndebug = "true";
+#else
+ const char* is_ndebug = "false";
+#endif
+
+ Value build_flags(kArrayType);
+ AddBuildFlag("is_ndebug", is_ndebug, document, &build_flags);
+ string cmake_build_type(GetCMakeBuildType());
+ replace(cmake_build_type.begin(), cmake_build_type.end(), '-', '_');
+ AddBuildFlag("cmake_build_type", cmake_build_type, document, &build_flags);
+ AddBuildFlag("library_link_type", GetLibraryLinkType(), document,
&build_flags);
+ document->AddMember("build_flags", build_flags, document->GetAllocator());
+
Value cpu_info(CpuInfo::DebugString().c_str(), document->GetAllocator());
document->AddMember("cpu_info", cpu_info, document->GetAllocator());
Value mem_info(MemInfo::DebugString().c_str(), document->GetAllocator());
http://git-wip-us.apache.org/repos/asf/impala/blob/9bdb73b1/bin/start-impala-cluster.py
----------------------------------------------------------------------
diff --git a/bin/start-impala-cluster.py b/bin/start-impala-cluster.py
index 7e22ca9..d052650 100755
--- a/bin/start-impala-cluster.py
+++ b/bin/start-impala-cluster.py
@@ -29,7 +29,7 @@ from getpass import getuser
from time import sleep, time
from optparse import OptionParser, SUPPRESS_HELP
from testdata.common import cgroups
-from tests.common.environ import specific_build_type_timeout
+from tests.common.environ import build_flavor_timeout
logging.basicConfig(level=logging.ERROR, format="%(asctime)s %(threadName)s:
%(message)s",
datefmt="%H:%M:%S")
@@ -120,7 +120,7 @@ CLUSTER_WAIT_TIMEOUT_IN_SECONDS = 240
# It is set to a high value to avoid failing if processes are slow to shut
down.
KILL_TIMEOUT_IN_SECONDS = 240
# For build types like ASAN, modify the default Kudu rpc timeout.
-KUDU_RPC_TIMEOUT = specific_build_type_timeout(0, slow_build_timeout=60000)
+KUDU_RPC_TIMEOUT = build_flavor_timeout(0, slow_build_timeout=60000)
def find_user_processes(binaries):
"""Returns an iterator over all processes owned by the current user with a
matching
http://git-wip-us.apache.org/repos/asf/impala/blob/9bdb73b1/tests/common/environ.py
----------------------------------------------------------------------
diff --git a/tests/common/environ.py b/tests/common/environ.py
index 092c2e0..e20aa98 100644
--- a/tests/common/environ.py
+++ b/tests/common/environ.py
@@ -15,20 +15,23 @@
# specific language governing permissions and limitations
# under the License.
+import json
import logging
import os
import re
+import requests
LOG = logging.getLogger('tests.common.environ')
test_start_cluster_args = os.environ.get("TEST_START_CLUSTER_ARGS", "")
IMPALA_HOME = os.environ.get("IMPALA_HOME", "")
+IMPALA_REMOTE_URL = os.environ.get("IMPALA_REMOTE_URL", "")
# Find the likely BuildType of the running Impala. Assume it's found through
the path
# $IMPALA_HOME/be/build/latest as a fallback.
build_type_arg_regex = re.compile(r'--build_type=(\w+)', re.I)
build_type_arg_search_result = re.search(build_type_arg_regex,
test_start_cluster_args)
if build_type_arg_search_result is not None:
- build_type_dir = build_type_search_result.groups()[0].lower()
+ build_type_dir = build_type_arg_search_result.groups()[0].lower()
else:
build_type_dir = 'latest'
@@ -36,9 +39,10 @@ else:
impalad_basedir = \
os.path.realpath(os.path.join(IMPALA_HOME, 'be/build',
build_type_dir)).rstrip('/')
-class SpecificImpaladBuildTypes:
+
+class ImpalaBuildFlavors:
"""
- Represents the possible CMAKE_BUILD_TYPE values. These specific build types
are needed
+ Represents the possible CMAKE_BUILD_TYPE values. These build flavors are
needed
by Python test code, e.g. to set different timeouts for different builds.
All values
are lower-cased to enable case-insensitive comparison.
"""
@@ -52,6 +56,8 @@ class SpecificImpaladBuildTypes:
CODE_COVERAGE_DEBUG = 'code_coverage_debug'
# ./buildall.sh -release -codecoverage
CODE_COVERAGE_RELEASE = 'code_coverage_release'
+ # ./buildall.sh -tidy
+ TIDY = 'tidy'
# ./buildall.sh -tsan
TSAN = 'tsan'
# ./buildall.sh -ubsan
@@ -60,82 +66,168 @@ class SpecificImpaladBuildTypes:
UBSAN_FULL = 'ubsan_full'
VALID_BUILD_TYPES = [ADDRESS_SANITIZER, DEBUG, CODE_COVERAGE_DEBUG, RELEASE,
- CODE_COVERAGE_RELEASE, TSAN, UBSAN, UBSAN_FULL]
+ CODE_COVERAGE_RELEASE, TIDY, TSAN, UBSAN, UBSAN_FULL]
+
+
+class LinkTypes:
+ """
+ Represents the possible library link type values, either "dynamic" or
"static". This
+ value is derived from the cmake value of BUILD_SHARED_LIBS. All values are
lower-cased
+ to enable case-insensitive comparison.
+ """
+ # ./buildall.sh
+ STATIC = 'static'
+ # ./buildall.sh -build_shared_libs
+ DYNAMIC = 'dynamic'
+
+ VALID_LINK_TYPES = [STATIC, DYNAMIC]
+
+
+class ImpalaTestClusterFlagsDetector:
+ """
+ Detects the build flags of different types of Impala clusters. Currently
supports
+ detecting build flags from either a locally built Impala cluster using a
file generated
+ by CMake, or from the Impala web ui, which is useful for detecting flags
from a remote
+ Impala cluster. The supported list of build flags is: [CMAKE_BUILD_TYPE,
+ BUILD_SHARED_LIBS]
+ """
@classmethod
- def detect(cls, impala_build_root):
+ def detect_using_build_root_or_web_ui(cls, impala_build_root):
"""
- Determine the build type based on the .cmake_build_type file created by
+ Determine the build flags based on the .cmake_build_type file created by
${IMPALA_HOME}/CMakeLists.txt. impala_build_root should be the path of the
- Impala source checkout, i.e. ${IMPALA_HOME}.
+ Impala source checkout, i.e. ${IMPALA_HOME}. If .cmake_build_type is not
present,
+ or cannot be read, attempt to detect the build flags from the local web UI
using
+ detect_using_web_ui.
"""
- build_type_path = os.path.join(impala_build_root, ".cmake_build_type")
+ cmake_build_type_path = os.path.join(impala_build_root,
".cmake_build_type")
try:
- with open(build_type_path) as build_type_file:
- build_type = build_type_file.read().strip().lower()
+ with open(cmake_build_type_path) as cmake_build_type_file:
+ build_flags = cmake_build_type_file.readlines()
+ build_type = build_flags[0].strip().lower()
+ build_shared_libs = build_flags[1].strip().lower()
except IOError:
- LOG.error("Could not open %s assuming DEBUG", build_type_path)
- return cls.DEBUG
+ LOG.debug("Unable to read .cmake_build_type file, fetching build flags
from " +
+ "web ui on localhost")
+ build_type, build_shared_libs =
ImpalaTestClusterFlagsDetector.detect_using_web_ui(
+ "http://localhost:25000";)
- if build_type not in cls.VALID_BUILD_TYPES:
+ library_link_type = LinkTypes.STATIC if build_shared_libs == "off"\
+ else LinkTypes.DYNAMIC
+ ImpalaTestClusterFlagsDetector.validate_build_flags(build_type,
library_link_type)
+ return build_type, library_link_type
+
+ @classmethod
+ def detect_using_web_ui(cls, impala_url):
+ """
+ Determine the build type based on the Impala cluster's web UI by using
+ get_build_flags_from_web_ui.
+ """
+ build_flags =
ImpalaTestClusterFlagsDetector.get_build_flags_from_web_ui(impala_url)
+ build_type = build_flags['cmake_build_type']
+ library_link_type = build_flags['library_link_type']
+ ImpalaTestClusterFlagsDetector.validate_build_flags(build_type,
library_link_type)
+ return build_type, library_link_type
+
+ @classmethod
+ def validate_build_flags(cls, build_type, library_link_type):
+ """
+ Validates that the build flags have valid values.
+ """
+ if build_type not in ImpalaBuildFlavors.VALID_BUILD_TYPES:
raise Exception("Unknown build type {0}".format(build_type))
+ if library_link_type not in LinkTypes.VALID_LINK_TYPES:
+ raise Exception("Unknown library link type
{0}".format(library_link_type))
LOG.debug("Build type detected: %s", build_type)
- return build_type
+ LOG.debug("Library link type detected: %s", library_link_type)
+
+ @classmethod
+ def get_build_flags_from_web_ui(cls, impala_url):
+ """
+ Fetches the build flags from the given Impala cluster web UI by parsing
the ?json
+ response of the root homepage and looking for the section on build flags.
It returns
+ the flags as a dictionary where the key is the flag name.
+ """
+ response = requests.get(impala_url + "/?json")
+ assert response.status_code == requests.codes.ok,\
+ "Offending url: " + impala_url
+ assert "application/json" in response.headers['Content-Type']
+
+ build_flags_json = json.loads(response.text)["build_flags"]
+ build_flags = dict((flag['flag_name'].lower(), flag['flag_value'].lower())
+ for flag in build_flags_json)
+ assert len(build_flags_json) == len(build_flags) # Ensure there are no
collisions
+ return build_flags
+"""
+Indicates whether we are operating against a locally built Impala cluster or a
remote one.
+"""
+(
+ LOCAL_BUILD,
+ REMOTE_BUILD,
+) = xrange(2)
-class ImpaladBuild(object):
+
+class ImpalaTestClusterProperties(object):
"""
Acquires and provides characteristics about the way the Impala under test
was compiled
- and its likely effects on its responsiveness to automated test timings.
Currently
- assumes that the Impala daemon under test was built in our current source
checkout.
- TODO: we could get this information for remote cluster tests if we exposed
the build
- type via a metric or the Impalad web UI.
+ and its likely effects on its responsiveness to automated test timings.
"""
- def __init__(self, impala_build_root):
- self._specific_build_type =
SpecificImpaladBuildTypes.detect(impala_build_root)
+ def __init__(self, build_flavor, library_link_type, local_or_remote_build):
+ self._build_flavor = build_flavor
+ self._library_link_type = library_link_type
+ self._local_or_remote_build = local_or_remote_build
+
+ @property
+ def build_flavor(self):
+ """
+ Return the correct ImpalaBuildFlavors for the Impala under test.
+ """
+ return self._build_flavor
@property
- def specific_build_type(self):
+ def library_link_type(self):
"""
- Return the correct SpecificImpaladBuildTypes for the Impala under test.
+ Return the library link type (either static or dynamic) for the Impala
under test.
"""
- return self._specific_build_type
+ return self._library_link_type
def has_code_coverage(self):
"""
Return whether the Impala under test was compiled with code coverage
enabled.
"""
- return self.specific_build_type in
(SpecificImpaladBuildTypes.CODE_COVERAGE_DEBUG,
-
SpecificImpaladBuildTypes.CODE_COVERAGE_RELEASE)
+ return self.build_flavor in (ImpalaBuildFlavors.CODE_COVERAGE_DEBUG,
+ ImpalaBuildFlavors.CODE_COVERAGE_RELEASE)
def is_asan(self):
"""
Return whether the Impala under test was compiled with ASAN.
"""
- return self.specific_build_type ==
SpecificImpaladBuildTypes.ADDRESS_SANITIZER
+ return self.build_flavor == ImpalaBuildFlavors.ADDRESS_SANITIZER
def is_tsan(self):
"""
Return whether the Impala under test was compiled with TSAN.
"""
- return self.specific_build_type == SpecificImpaladBuildTypes.TSAN
+ return self.build_flavor == ImpalaBuildFlavors.TSAN
def is_ubsan(self):
"""
Return whether the Impala under test was compiled with UBSAN.
"""
- return self.specific_build_type == SpecificImpaladBuildTypes.UBSAN
+ return self.build_flavor == ImpalaBuildFlavors.UBSAN
def is_dev(self):
"""
Return whether the Impala under test is a development build (i.e., any
debug or ASAN
build).
"""
- return self.specific_build_type in (
- SpecificImpaladBuildTypes.ADDRESS_SANITIZER,
SpecificImpaladBuildTypes.DEBUG,
- SpecificImpaladBuildTypes.CODE_COVERAGE_DEBUG,
- SpecificImpaladBuildTypes.TSAN, SpecificImpaladBuildTypes.UBSAN)
+ return self.build_flavor in (
+ ImpalaBuildFlavors.ADDRESS_SANITIZER, ImpalaBuildFlavors.DEBUG,
+ ImpalaBuildFlavors.CODE_COVERAGE_DEBUG, ImpalaBuildFlavors.TSAN,
+ ImpalaBuildFlavors.UBSAN)
def runs_slowly(self):
"""
@@ -144,15 +236,42 @@ class ImpaladBuild(object):
"""
return self.has_code_coverage() or self.is_asan() or self.is_tsan() or
self.is_ubsan()
+ def is_statically_linked(self):
+ """
+ Return whether the Impala under test was statically linked during
compilation.
+ """
+ return self.build_shared_libs == LinkTypes.STATIC
+
+ def is_dynamically_linked(self):
+ """
+ Return whether the Impala under test was dynamically linked during
compilation.
+ """
+ return self.build_shared_libs == LinkTypes.DYNAMIC
+
+ def is_remote_cluster(self):
+ """
+ Return true if the Impala test cluster is running remotely, false otherwise
+ """
+ return self._local_or_remote_build == REMOTE_BUILD
+
-IMPALAD_BUILD = ImpaladBuild(IMPALA_HOME)
+if IMPALA_REMOTE_URL:
+ build_flavor, link_type =\
+ ImpalaTestClusterFlagsDetector.detect_using_web_ui(IMPALA_REMOTE_URL)
+ IMPALA_TEST_CLUSTER_PROPERTIES =\
+ ImpalaTestClusterProperties(build_flavor, link_type, REMOTE_BUILD)
+else:
+ build_flavor, link_type =\
+
ImpalaTestClusterFlagsDetector.detect_using_build_root_or_web_ui(IMPALA_HOME)
+ IMPALA_TEST_CLUSTER_PROPERTIES =\
+ ImpalaTestClusterProperties(build_flavor, link_type, LOCAL_BUILD)
-def specific_build_type_timeout(
- default_timeout, slow_build_timeout=None, asan_build_timeout=None,
- code_coverage_build_timeout=None):
+
+def build_flavor_timeout(default_timeout, slow_build_timeout=None,
+ asan_build_timeout=None, code_coverage_build_timeout=None):
"""
- Return a test environment-specific timeout based on the sort of
- SpecificImpalaBuildType under test.
+ Return a test environment-specific timeout based on the sort of
ImpalaBuildFlavor under
+ test.
Required parameter: default_timeout - default timeout value. This applies
when Impala is
a standard release or debug build, or if no other timeouts are specified.
@@ -163,7 +282,7 @@ def specific_build_type_timeout(
"slow". You can use this as a shorthand in lieu of specifying all of the
following
parameters.
- The parameters below correspond to specific build types. These preempt both
+ The parameters below correspond to build flavors. These preempt both
slow_build_timeout and default_timeout, if the Impala under test is a build
of the
applicable type:
@@ -173,13 +292,13 @@ def specific_build_type_timeout(
(both debug and release code coverage)
"""
- if IMPALAD_BUILD.is_asan() and asan_build_timeout is not None:
+ if IMPALA_TEST_CLUSTER_PROPERTIES.is_asan() and asan_build_timeout is not
None:
timeout_val = asan_build_timeout
- elif IMPALAD_BUILD.has_code_coverage() and code_coverage_build_timeout is
not None:
+ elif IMPALA_TEST_CLUSTER_PROPERTIES.has_code_coverage() and\
+ code_coverage_build_timeout is not None:
timeout_val = code_coverage_build_timeout
- elif IMPALAD_BUILD.runs_slowly() and slow_build_timeout is not None:
+ elif IMPALA_TEST_CLUSTER_PROPERTIES.runs_slowly() and slow_build_timeout is
not None:
timeout_val = slow_build_timeout
else:
timeout_val = default_timeout
return timeout_val
-
http://git-wip-us.apache.org/repos/asf/impala/blob/9bdb73b1/tests/common/skip.py
----------------------------------------------------------------------
diff --git a/tests/common/skip.py b/tests/common/skip.py
index 2f19bf4..41e119a 100644
--- a/tests/common/skip.py
+++ b/tests/common/skip.py
@@ -24,7 +24,7 @@ import os
import pytest
from functools import partial
-from tests.common.environ import IMPALAD_BUILD
+from tests.common.environ import IMPALA_TEST_CLUSTER_PROPERTIES
from tests.util.filesystem_utils import (
IS_ABFS,
IS_ADLS,
@@ -161,8 +161,10 @@ class SkipIfNotHdfsMinicluster:
reason="Test is tuned for 3-node HDFS minicluster with no EC")
class SkipIfBuildType:
- not_dev_build = pytest.mark.skipif(not IMPALAD_BUILD.is_dev(),
- reason="Tests depends on debug build startup option.")
+ not_dev_build = pytest.mark.skipif(not
IMPALA_TEST_CLUSTER_PROPERTIES.is_dev(),
+ reason="Test depends on debug build startup option.")
+ remote =
pytest.mark.skipif(IMPALA_TEST_CLUSTER_PROPERTIES.is_remote_cluster(),
+ reason="Test depends on running against a local Impala cluster")
class SkipIfEC:
remote_read = pytest.mark.skipif(IS_EC, reason="EC files are read remotely
and "
http://git-wip-us.apache.org/repos/asf/impala/blob/9bdb73b1/tests/conftest.py
----------------------------------------------------------------------
diff --git a/tests/conftest.py b/tests/conftest.py
index 4e1c837..0fccc5d 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -27,7 +27,7 @@ import logging
import os
import pytest
-from tests.common.environ import specific_build_type_timeout
+from tests.common.environ import build_flavor_timeout
from common.test_result_verifier import QueryTestResult
from tests.common.patterns import is_valid_impala_identifier
from tests.comparison.db_connection import ImpalaConnection
@@ -52,7 +52,7 @@ if FILESYSTEM == 'isilon':
# Timeout each individual test case after 2 hours, or 4 hours for slow builds
PYTEST_TIMEOUT_S = \
- specific_build_type_timeout(2 * 60 * 60, slow_build_timeout=4 * 60 * 60)
+ build_flavor_timeout(2 * 60 * 60, slow_build_timeout=4 * 60 * 60)
def pytest_configure(config):
""" Hook startup of pytest. Sets up log format and per-test timeout. """
http://git-wip-us.apache.org/repos/asf/impala/blob/9bdb73b1/tests/custom_cluster/test_admission_controller.py
----------------------------------------------------------------------
diff --git a/tests/custom_cluster/test_admission_controller.py
b/tests/custom_cluster/test_admission_controller.py
index 6b3100f..f716788 100644
--- a/tests/custom_cluster/test_admission_controller.py
+++ b/tests/custom_cluster/test_admission_controller.py
@@ -31,7 +31,7 @@ from time import sleep, time
from beeswaxd.BeeswaxService import QueryState
from tests.beeswax.impala_beeswax import ImpalaBeeswaxException
from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
-from tests.common.environ import specific_build_type_timeout, IMPALAD_BUILD
+from tests.common.environ import build_flavor_timeout,
IMPALA_TEST_CLUSTER_PROPERTIES
from tests.common.impala_test_suite import ImpalaTestSuite
from tests.common.resource_pool_config import ResourcePoolConfig
from tests.common.skip import (
@@ -119,7 +119,7 @@ POOL_NAME = "default-pool"
# Stress test timeout (seconds). The timeout needs to be significantly higher
for
# slow builds like code coverage and ASAN (IMPALA-3790, IMPALA-6241).
-STRESS_TIMEOUT = specific_build_type_timeout(60, slow_build_timeout=600)
+STRESS_TIMEOUT = build_flavor_timeout(60, slow_build_timeout=600)
# The number of queries that can execute concurrently in the pool POOL_NAME.
MAX_NUM_CONCURRENT_QUERIES = 5
@@ -902,7 +902,7 @@ class
TestAdmissionControllerStress(TestAdmissionControllerBase):
# Additional constraints for code coverage jobs and core.
num_queries = None
- if IMPALAD_BUILD.has_code_coverage():
+ if IMPALA_TEST_CLUSTER_PROPERTIES.has_code_coverage():
# Code coverage builds can't handle the increased concurrency.
num_queries = 15
elif cls.exploration_strategy() == 'core':
http://git-wip-us.apache.org/repos/asf/impala/blob/9bdb73b1/tests/custom_cluster/test_automatic_invalidation.py
----------------------------------------------------------------------
diff --git a/tests/custom_cluster/test_automatic_invalidation.py
b/tests/custom_cluster/test_automatic_invalidation.py
index 019712b..e82217b 100644
--- a/tests/custom_cluster/test_automatic_invalidation.py
+++ b/tests/custom_cluster/test_automatic_invalidation.py
@@ -19,7 +19,7 @@ import os
import pytest
import time
from subprocess import call
-from tests.common.environ import IMPALAD_BUILD
+from tests.common.environ import IMPALA_TEST_CLUSTER_PROPERTIES
from tests.util.filesystem_utils import IS_HDFS, IS_LOCAL
@@ -37,7 +37,8 @@ class
TestAutomaticCatalogInvalidation(CustomClusterTestSuite):
# The test will run a query and assumes the table is loaded when the query
finishes.
# The timeout should be larger than the time of the query.
- timeout = 20 if IMPALAD_BUILD.runs_slowly() or (not IS_HDFS and not
IS_LOCAL) else 10
+ timeout = 20 if IMPALA_TEST_CLUSTER_PROPERTIES.runs_slowly() or\
+ (not IS_HDFS and not IS_LOCAL) else 10
timeout_flag = "--invalidate_tables_timeout_s=" + str(timeout)
@classmethod
http://git-wip-us.apache.org/repos/asf/impala/blob/9bdb73b1/tests/custom_cluster/test_exchange_delays.py
----------------------------------------------------------------------
diff --git a/tests/custom_cluster/test_exchange_delays.py
b/tests/custom_cluster/test_exchange_delays.py
index 1d7d14b..38dfde1 100644
--- a/tests/custom_cluster/test_exchange_delays.py
+++ b/tests/custom_cluster/test_exchange_delays.py
@@ -17,13 +17,13 @@
import pytest
from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
-from tests.common.environ import specific_build_type_timeout
+from tests.common.environ import build_flavor_timeout
from tests.common.skip import SkipIfBuildType
from tests.util.filesystem_utils import IS_S3, IS_ADLS, IS_ISILON
# IMPALA-6100: add additional margin for error for slow build types.
SLOW_BUILD_TIMEOUT=20000
-DELAY_MS = specific_build_type_timeout(10000,
slow_build_timeout=SLOW_BUILD_TIMEOUT)
+DELAY_MS = build_flavor_timeout(10000, slow_build_timeout=SLOW_BUILD_TIMEOUT)
# IMPALA-6381: Isilon can behave as a slow build.
if IS_ISILON:
DELAY_MS = SLOW_BUILD_TIMEOUT
http://git-wip-us.apache.org/repos/asf/impala/blob/9bdb73b1/tests/custom_cluster/test_metadata_replicas.py
----------------------------------------------------------------------
diff --git a/tests/custom_cluster/test_metadata_replicas.py
b/tests/custom_cluster/test_metadata_replicas.py
index 940f371..ff34cf2 100644
--- a/tests/custom_cluster/test_metadata_replicas.py
+++ b/tests/custom_cluster/test_metadata_replicas.py
@@ -18,7 +18,6 @@
import pytest
import re
from time import sleep
-from tests.common.environ import specific_build_type_timeout
from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
from tests.common.skip import (
SkipIfS3,
http://git-wip-us.apache.org/repos/asf/impala/blob/9bdb73b1/tests/custom_cluster/test_restart_services.py
----------------------------------------------------------------------
diff --git a/tests/custom_cluster/test_restart_services.py
b/tests/custom_cluster/test_restart_services.py
index a24149f..e441cbc 100644
--- a/tests/custom_cluster/test_restart_services.py
+++ b/tests/custom_cluster/test_restart_services.py
@@ -22,7 +22,7 @@ import re
import socket
import time
-from tests.common.environ import specific_build_type_timeout
+from tests.common.environ import build_flavor_timeout
from time import sleep
from impala.error import HiveServer2Error
@@ -50,7 +50,7 @@ class TestRestart(CustomClusterTestSuite):
# We need to wait for the impalad to register to the new statestored and
for a
# non-empty catalog update from the new statestored. It cannot be
expressed with the
# existing metrics yet so we wait for some time here.
- wait_time_s = specific_build_type_timeout(60, slow_build_timeout=100)
+ wait_time_s = build_flavor_timeout(60, slow_build_timeout=100)
sleep(wait_time_s)
for retry in xrange(wait_time_s):
try:
http://git-wip-us.apache.org/repos/asf/impala/blob/9bdb73b1/tests/query_test/test_hdfs_caching.py
----------------------------------------------------------------------
diff --git a/tests/query_test/test_hdfs_caching.py
b/tests/query_test/test_hdfs_caching.py
index 176fe09..64b36da 100644
--- a/tests/query_test/test_hdfs_caching.py
+++ b/tests/query_test/test_hdfs_caching.py
@@ -22,7 +22,7 @@ import re
import time
from subprocess import check_call
-from tests.common.environ import specific_build_type_timeout
+from tests.common.environ import build_flavor_timeout
from tests.common.impala_cluster import ImpalaCluster
from tests.common.impala_test_suite import ImpalaTestSuite, LOG
from tests.common.skip import SkipIfS3, SkipIfABFS, SkipIfADLS, SkipIfIsilon, \
@@ -339,7 +339,7 @@ def get_num_cache_requests():
return len(stdout.split('\n'))
# IMPALA-3040: This can take time, especially under slow builds like ASAN.
- wait_time_in_sec = specific_build_type_timeout(5, slow_build_timeout=20)
+ wait_time_in_sec = build_flavor_timeout(5, slow_build_timeout=20)
num_stabilization_attempts = 0
max_num_stabilization_attempts = 10
new_requests = None
http://git-wip-us.apache.org/repos/asf/impala/blob/9bdb73b1/tests/query_test/test_runtime_filters.py
----------------------------------------------------------------------
diff --git a/tests/query_test/test_runtime_filters.py
b/tests/query_test/test_runtime_filters.py
index 547a1ee..f769224 100644
--- a/tests/query_test/test_runtime_filters.py
+++ b/tests/query_test/test_runtime_filters.py
@@ -20,11 +20,11 @@ import pytest
import re
import time
-from tests.common.environ import specific_build_type_timeout
+from tests.common.environ import build_flavor_timeout
from tests.common.impala_test_suite import ImpalaTestSuite
from tests.common.skip import SkipIfLocal, SkipIfIsilon
-WAIT_TIME_MS = specific_build_type_timeout(60000, slow_build_timeout=100000)
+WAIT_TIME_MS = build_flavor_timeout(60000, slow_build_timeout=100000)
# Some of the queries in runtime_filters consume a lot of memory, leading to
# significant memory reservations in parallel tests.
http://git-wip-us.apache.org/repos/asf/impala/blob/9bdb73b1/tests/run-tests.py
----------------------------------------------------------------------
diff --git a/tests/run-tests.py b/tests/run-tests.py
index 9aaeaa5..0de9ce9 100755
--- a/tests/run-tests.py
+++ b/tests/run-tests.py
@@ -37,7 +37,17 @@ from _pytest.config import FILE_OR_DIR
# We whitelist valid test directories. If a new test directory is added,
update this.
VALID_TEST_DIRS = ['failure', 'query_test', 'stress', 'unittests',
'aux_query_tests',
'shell', 'hs2', 'catalog_service', 'metadata',
'data_errors',
- 'statestore', 'infra', 'observability']
+ 'statestore', 'infra', 'observability', 'webserver']
+
+# A list of helper directories that do not contain any tests. The purpose of
this
+# additional list is to prevent devs from adding a new test dir, but not
adding the
+# new dir to the list of valid test dirs above. All dirs unders tests/ must be
placed
+# into one of these lists, otherwise the script will throw an error. This list
can be
+# removed once IMPALA-4417 has been resolved.
+TEST_HELPER_DIRS = ['aux_parquet_data_load', 'test-hive-udfs', 'comparison',
'benchmark',
+ 'custom_cluster', 'util', 'experiments', 'verifiers',
'common',
+ 'performance', 'beeswax', 'aux_custom_cluster_tests',
+ 'authorization']
TEST_DIR = os.path.join(os.environ['IMPALA_HOME'], 'tests')
RESULT_DIR = os.path.join(os.environ['IMPALA_EE_TEST_LOGS_DIR'], 'results')
@@ -201,7 +211,12 @@ def build_ignore_dir_arg_list(valid_dirs):
code as though it contained tests, resulting in misleading warnings or
failures.
(There is a JIRA filed to restructure this: IMPALA-4417.)
"""
- subdirs = [subdir for subdir in os.listdir(TEST_DIR) if
os.path.isdir(subdir)]
+ subdirs = [subdir for subdir in os.listdir(TEST_DIR)
+ if os.path.isdir(subdir) and not subdir.startswith(".")]
+ for subdir in subdirs:
+ assert subdir in VALID_TEST_DIRS or subdir in TEST_HELPER_DIRS,\
+ "Unexpected test dir '%s' is not in the list of valid or helper test
dirs"\
+ % subdir
ignored_dir_list = []
for subdir in (set(subdirs) - set(valid_dirs)):
ignored_dir_list += ['--ignore', subdir]
http://git-wip-us.apache.org/repos/asf/impala/blob/9bdb73b1/tests/statestore/test_statestore.py
----------------------------------------------------------------------
diff --git a/tests/statestore/test_statestore.py
b/tests/statestore/test_statestore.py
index f6a6bb2..a951414 100644
--- a/tests/statestore/test_statestore.py
+++ b/tests/statestore/test_statestore.py
@@ -39,7 +39,7 @@ from StatestoreService.StatestoreSubscriber import
TTopicRegistration
from ErrorCodes.ttypes import TErrorCode
from Status.ttypes import TStatus
-from tests.common.environ import specific_build_type_timeout
+from tests.common.environ import build_flavor_timeout
LOG = logging.getLogger('test_statestore')
@@ -69,10 +69,10 @@ DEFAULT_UPDATE_STATE_RESPONSE =
TUpdateStateResponse(status=STATUS_OK, topic_upd
skipped=False)
# IMPALA-3501: the timeout needs to be higher in code coverage builds
-WAIT_FOR_FAILURE_TIMEOUT = specific_build_type_timeout(40,
code_coverage_build_timeout=60)
-WAIT_FOR_HEARTBEAT_TIMEOUT = specific_build_type_timeout(
+WAIT_FOR_FAILURE_TIMEOUT = build_flavor_timeout(40,
code_coverage_build_timeout=60)
+WAIT_FOR_HEARTBEAT_TIMEOUT = build_flavor_timeout(
40, code_coverage_build_timeout=60)
-WAIT_FOR_UPDATE_TIMEOUT = specific_build_type_timeout(40,
code_coverage_build_timeout=60)
+WAIT_FOR_UPDATE_TIMEOUT = build_flavor_timeout(40,
code_coverage_build_timeout=60)
class WildcardServerSocket(TSocket.TSocketBase,
TTransport.TServerTransportBase):
"""Specialised server socket that binds to a random port at construction"""
http://git-wip-us.apache.org/repos/asf/impala/blob/9bdb73b1/tests/webserver/test_web_pages.py
----------------------------------------------------------------------
diff --git a/tests/webserver/test_web_pages.py
b/tests/webserver/test_web_pages.py
index 15d4f8f..0c05546 100644
--- a/tests/webserver/test_web_pages.py
+++ b/tests/webserver/test_web_pages.py
@@ -15,14 +15,19 @@
# specific language governing permissions and limitations
# under the License.
-from tests.common.skip import SkipIf
+from tests.common.environ import IMPALA_TEST_CLUSTER_PROPERTIES
+from tests.common.environ import ImpalaTestClusterFlagsDetector
+from tests.common.skip import SkipIfBuildType
from tests.common.impala_cluster import ImpalaCluster
from tests.common.impala_test_suite import ImpalaTestSuite
import json
import requests
+import pytest
+
class TestWebPage(ImpalaTestSuite):
+ ROOT_URL = "http://localhost:{0}/";
GET_JAVA_LOGLEVEL_URL = "http://localhost:{0}/get_java_loglevel";
SET_JAVA_LOGLEVEL_URL = "http://localhost:{0}/set_java_loglevel";
RESET_JAVA_LOGLEVEL_URL = "http://localhost:{0}/reset_java_loglevel";
@@ -46,8 +51,55 @@ class TestWebPage(ImpalaTestSuite):
TEST_PORTS_WITH_SS = ["25000", "25010", "25020"]
CATALOG_TEST_PORT = ["25020"]
+ def test_get_root_url(self):
+ """Tests that the root URL is accessible and loads properly"""
+ self.get_and_check_status(self.ROOT_URL)
+
+ def test_get_build_flags(self):
+ """Tests that the build flags on the root page contain valid values"""
+ for port in self.TEST_PORTS_WITH_SS:
+ build_flags = ImpalaTestClusterFlagsDetector.\
+ get_build_flags_from_web_ui(self.ROOT_URL.format(port))
+
+ assert len(build_flags) == 3
+ assert "is_ndebug" in build_flags
+ assert build_flags["is_ndebug"] in ["true", "false"]
+ assert "cmake_build_type" in build_flags
+ assert build_flags["cmake_build_type"] in ["debug", "release",
"address_sanitizer",
+ "tidy", "ubsan", "ubsan_full", "tsan", "code_coverage_release",
+ "code_coverage_debug"]
+ assert "library_link_type" in build_flags
+ assert build_flags["library_link_type"] in ["dynamic", "static"]
+
+ @SkipIfBuildType.remote
+ def test_root_correct_build_flags(self):
+ """Tests that the build flags on the root page contain correct values"""
+ assert not IMPALA_TEST_CLUSTER_PROPERTIES.is_remote_cluster()
+ for port in self.TEST_PORTS_WITH_SS:
+ build_flags = ImpalaTestClusterFlagsDetector.\
+ get_build_flags_from_web_ui(self.ROOT_URL.format(port))
+
+ assert build_flags["cmake_build_type"] ==\
+ IMPALA_TEST_CLUSTER_PROPERTIES.build_flavor
+ assert build_flags["library_link_type"] ==\
+ IMPALA_TEST_CLUSTER_PROPERTIES.library_link_type
+
+ def test_root_consistent_build_flags(self):
+ """Tests that the build flags on the root page contain consistent values"""
+ for port in self.TEST_PORTS_WITH_SS:
+ build_flags = ImpalaTestClusterFlagsDetector.\
+ get_build_flags_from_web_ui(self.ROOT_URL.format(port))
+
+ is_ndebug = build_flags["is_ndebug"] == "true"
+
+ if not is_ndebug:
+ assert not build_flags["cmake_build_type"] in ["release"]
+
+ if build_flags["cmake_build_type"] in ["debug"]:
+ assert not is_ndebug
+
def test_memz(self):
- """test /memz at impalad / statestored / catalogd"""
+ """Tests /memz at impalad / statestored / catalogd"""
page = requests.get("http://localhost:25000/memz";)
assert page.status_code == requests.codes.ok
@@ -79,26 +131,31 @@ class TestWebPage(ImpalaTestSuite):
except ValueError:
assert False, "Invalid JSON returned from /jmx endpoint: %s" % jmx_json
- def get_and_check_status(self, url, string_to_search = "", ports_to_test =
None):
+ def get_and_check_status(self, url, string_to_search="", ports_to_test=None):
"""Helper method that polls a given url and asserts the return code is ok
and
the response contains the input string."""
if ports_to_test is None:
ports_to_test = self.TEST_PORTS_WITH_SS
+
+ responses = []
for port in ports_to_test:
input_url = url.format(port)
response = requests.get(input_url)
assert response.status_code == requests.codes.ok\
and string_to_search in response.text, "Offending url: " + input_url
+ responses.append(response)
+ return responses
- def get_debug_page(self, page_url):
+ def get_debug_page(self, page_url, port=25000):
"""Returns the content of the debug page 'page_url' as json."""
- response = self.get_and_check_status(page_url + "?json",
ports_to_test=[25000])
- assert "application/json" in response.headers['Content-Type']
- return json.loads(response)
+ responses = self.get_and_check_status(page_url + "?json",
ports_to_test=[port])
+ assert len(responses) == 1
+ assert "application/json" in responses[0].headers['Content-Type']
+ return json.loads(responses[0].text)
- def get_and_check_status_jvm(self, url, string_to_search = ""):
+ def get_and_check_status_jvm(self, url, string_to_search=""):
"""Calls get_and_check_status() for impalad and catalogd only"""
- self.get_and_check_status(url, string_to_search,
+ return self.get_and_check_status(url, string_to_search,
ports_to_test=self.TEST_PORTS_WITHOUT_SS)
def test_content_type(self):
@@ -151,7 +208,7 @@ class TestWebPage(ImpalaTestSuite):
self.get_and_check_status(set_glog_url, "v set to 3")
# Try resetting the glog logging defaults again.
- self.get_and_check_status( self.RESET_GLOG_LOGLEVEL_URL, "v set to ")
+ self.get_and_check_status(self.RESET_GLOG_LOGLEVEL_URL, "v set to ")
# Try to get the log level of an empty class input
get_loglevel_url = (self.GET_JAVA_LOGLEVEL_URL + "?class=")
@@ -214,17 +271,19 @@ class TestWebPage(ImpalaTestSuite):
cancels the query."""
if query_options:
self.client.set_configuration(query_options)
- query_handle = self.client.execute_async(query)
+ query_handle = self.client.execute_async(query)
response_json = ""
try:
- response = self.get_and_check_status(
+ responses = self.get_and_check_status(
page_url + "?query_id=%s&json" % query_handle.get_handle().id,
ports_to_test=[25000])
- response_json = json.loads(response)
+ assert len(responses) == 1
+ response_json = json.loads(responses[0].text)
finally:
self.client.cancel(query_handle)
return response_json
+ @pytest.mark.skip(reason='IMPALA-7625')
def test_backend_states(self, unique_database):
"""Test that /query_backends returns the list of backend states for DML or
queries;
nothing for DDL statements"""
@@ -240,6 +299,7 @@ class TestWebPage(ImpalaTestSuite):
else:
assert 'backend_states' not in response_json
+ @pytest.mark.skip(reason='IMPALA-7625')
def test_backend_instances(self, unique_database):
"""Test that /query_finstances returns the list of fragment instances for
DML or
queries; nothing for DDL statements"""
@@ -255,6 +315,7 @@ class TestWebPage(ImpalaTestSuite):
else:
assert 'backend_instances' not in response_json
+ @pytest.mark.skip(reason='IMPALA-7625')
def test_backend_instances_mt_dop(self, unique_database):
"""Test that accessing /query_finstances does not crash the backend when
running with
mt_dop."""
@@ -268,14 +329,15 @@ class TestWebPage(ImpalaTestSuite):
def test_io_mgr_threads(self):
"""Test that IoMgr threads have readable names. This test assumed that all
systems we
support have a disk called 'sda'."""
- response = self.get_and_check_status(
+ responses = self.get_and_check_status(
self.THREAD_GROUP_URL + "?group=disk-io-mgr&json",
ports_to_test=[25000])
- response_json = json.loads(response)
+ assert len(responses) == 1
+ response_json = json.loads(responses[0].text)
thread_names = [t["name"] for t in response_json['threads']]
- expected_name_patterns = ["ADLS remote", "S3 remote", "HDFS remote", "sda"]
+ expected_name_patterns = ["ADLS remote", "S3 remote", "HDFS remote"]
for pattern in expected_name_patterns:
assert any(pattern in t for t in thread_names), \
- "Could not find thread matching '%s'" % pattern
+ "Could not find thread matching '%s'" % pattern
def test_krpc_rpcz(self):
"""Test that KRPC metrics are exposed in /rpcz and that they are updated
when
http://git-wip-us.apache.org/repos/asf/impala/blob/9bdb73b1/www/root.tmpl
----------------------------------------------------------------------
diff --git a/www/root.tmpl b/www/root.tmpl
index 7d156c4..9d73c41 100644
--- a/www/root.tmpl
+++ b/www/root.tmpl
@@ -32,7 +32,10 @@ under the License.
{{/impala_server_mode}}
<h2>Vers<span id="v">i</span>on</h2>
- <pre id="version_pre">{{version}}</pre>
+ <pre id="version_pre">{{version}}
+Build Flags: {{#build_flags}}{{flag_name}}={{flag_value}}
{{/build_flags}}</pre>
+<!-- The space after {{flag_value}} is necessary to add a space between each
flag when
+this page is rendered -->
<h2>Process Start Time</h2>
<pre>{{process_start_time}}</pre>
