This is an automated email from the ASF dual-hosted git repository. michaelsmith pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/impala.git
commit 88d49b6919501f37e0cc5633d5d9046f526b66b8 Author: Michael Smith <[email protected]> AuthorDate: Wed Dec 14 15:18:27 2022 -0800 IMPALA-11693: Enable allow_erasure_coded_files by default Enables allow_erasure_coded_files by default as we've now completed all planned work to support it. Testing - Ran HDFS+EC test suite - Ran Ozone+EC test suite Change-Id: I0cfef087f2a7ae0889f47e85c5fab61a795d8fd4 Reviewed-on: http://gerrit.cloudera.org:8080/19362 Reviewed-by: Joe McDonnell <[email protected]> Tested-by: Impala Public Jenkins <[email protected]> --- bin/run-all-tests.sh | 2 -- common/thrift/Query.thrift | 2 +- docs/topics/impala_allow_erasure_coded_files.xml | 5 ++--- testdata/bin/create-load-data.sh | 4 ---- tests/common/custom_cluster_test_suite.py | 2 -- tests/query_test/test_observability.py | 6 ++---- tests/util/auto_scaler.py | 2 -- 7 files changed, 5 insertions(+), 18 deletions(-) diff --git a/bin/run-all-tests.sh b/bin/run-all-tests.sh index 7877766a4..ca45f6a61 100755 --- a/bin/run-all-tests.sh +++ b/bin/run-all-tests.sh @@ -94,8 +94,6 @@ if [[ "${ERASURE_CODING}" = true ]]; then # We do not run FE tests when erasure coding is enabled because planner tests # would fail. FE_TEST=false - TEST_START_CLUSTER_ARGS="${TEST_START_CLUSTER_ARGS} \ - --impalad_args=--default_query_options=allow_erasure_coded_files=true" fi # Indicates whether code coverage reports should be generated. diff --git a/common/thrift/Query.thrift b/common/thrift/Query.thrift index 80230f5f9..2c433b809 100644 --- a/common/thrift/Query.thrift +++ b/common/thrift/Query.thrift @@ -339,7 +339,7 @@ struct TQueryOptions { 68: optional TKuduReadMode kudu_read_mode = TKuduReadMode.DEFAULT; // Allow reading of erasure coded files in HDFS. - 69: optional bool allow_erasure_coded_files = false; + 69: optional bool allow_erasure_coded_files = true; // See comment in ImpalaService.thrift. 70: optional string timezone = "" diff --git a/docs/topics/impala_allow_erasure_coded_files.xml b/docs/topics/impala_allow_erasure_coded_files.xml index b3089ee71..30228e924 100644 --- a/docs/topics/impala_allow_erasure_coded_files.xml +++ b/docs/topics/impala_allow_erasure_coded_files.xml @@ -40,8 +40,7 @@ under the License. <p> Use the <codeph>ALLOW_ERASURE_CODED_FILES</codeph> query option to enable or disable the - support of erasure coded files in Impala. Until Impala is fully tested and certified with - erasure coded files, this query option is set to <codeph>FALSE</codeph> by default. + support of erasure coded files in Impala. </p> <p> @@ -55,7 +54,7 @@ under the License. </p> <p> - <b>Default:</b> <codeph>FALSE</codeph> + <b>Default:</b> <codeph>TRUE</codeph> </p> <p> diff --git a/testdata/bin/create-load-data.sh b/testdata/bin/create-load-data.sh index e13f1302c..8b5f67859 100755 --- a/testdata/bin/create-load-data.sh +++ b/testdata/bin/create-load-data.sh @@ -169,10 +169,6 @@ function start-impala { else START_CLUSTER_ARGS_INT+=("-s 3") fi - if [[ "${ERASURE_CODING}" == true ]]; then - START_CLUSTER_ARGS="${START_CLUSTER_ARGS} \ - --impalad_args=--default_query_options=allow_erasure_coded_files=true" - fi START_CLUSTER_ARGS_INT+=("${START_CLUSTER_ARGS}") ${IMPALA_HOME}/bin/start-impala-cluster.py --log_dir=${IMPALA_DATA_LOADING_LOGS_DIR} \ ${START_CLUSTER_ARGS_INT[@]} diff --git a/tests/common/custom_cluster_test_suite.py b/tests/common/custom_cluster_test_suite.py index 65a2fe8d5..bc940cd0b 100644 --- a/tests/common/custom_cluster_test_suite.py +++ b/tests/common/custom_cluster_test_suite.py @@ -315,8 +315,6 @@ class CustomClusterTestSuite(ImpalaTestSuite): default_query_option_kvs = [] # Put any defaults first, then any arguments after that so they can override defaults. - if os.environ.get("ERASURE_CODING") == "true": - default_query_option_kvs.append(("allow_erasure_coded_files", "true")) if default_query_options is not None: default_query_option_kvs.extend(default_query_options) # Add the default query options after any arguments. This will override any default diff --git a/tests/query_test/test_observability.py b/tests/query_test/test_observability.py index 25bc2347e..51124bb61 100644 --- a/tests/query_test/test_observability.py +++ b/tests/query_test/test_observability.py @@ -178,14 +178,12 @@ class TestObservability(ImpalaTestSuite): expected_str = ("Query Options (set by configuration and planner): " "MEM_LIMIT=8589934592," "NUM_NODES=1,NUM_SCANNER_THREADS=1," - "RUNTIME_FILTER_MODE=OFF,MT_DOP=0,{erasure_coding}TIMEZONE={timezone}," + "RUNTIME_FILTER_MODE=OFF,MT_DOP=0,TIMEZONE={timezone}," "CLIENT_IDENTIFIER=" "query_test/test_observability.py::TestObservability::()::test_query_options," "SPOOL_QUERY_RESULTS=0" "\n") - expected_str = expected_str.format( - erasure_coding="ALLOW_ERASURE_CODED_FILES=1," if IS_EC else "", - timezone=server_timezone) + expected_str = expected_str.format(timezone=server_timezone) assert expected_str in profile, profile def test_exec_summary(self): diff --git a/tests/util/auto_scaler.py b/tests/util/auto_scaler.py index e9754beb6..1236aaa84 100755 --- a/tests/util/auto_scaler.py +++ b/tests/util/auto_scaler.py @@ -256,8 +256,6 @@ class AutoScaler(object): "-vmodule=admission-controller=3,cluster-membership-mgr=3", "-admission_control_slots=%s" % executor_slots, "-shutdown_grace_period_s=2"] - if IS_EC: - impalad_args.append("--default_query_options=allow_erasure_coded_files=true") options += ["--impalad_args=%s" % a for a in impalad_args]
