IMPALA-5888: free other local allocations in Parquet Testing: I wasn't able to produce any abnormal memory consumption from dictionary or min/max filters so haven't included a regression test.
Change-Id: I7792552510b54aa95044e44218e3351a36d6f9a8 Reviewed-on: http://gerrit.cloudera.org:8080/7933 Reviewed-by: Tim Armstrong <[email protected]> Tested-by: Impala Public Jenkins Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/be98aaac Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/be98aaac Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/be98aaac Branch: refs/heads/master Commit: be98aaacada6966ca39741696d8f95d24f2d3117 Parents: 4c9b46a Author: Tim Armstrong <[email protected]> Authored: Thu Aug 31 22:19:48 2017 -0700 Committer: Impala Public Jenkins <[email protected]> Committed: Thu Sep 7 00:52:56 2017 +0000 ---------------------------------------------------------------------- be/src/exec/hdfs-parquet-scanner.cc | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/be98aaac/be/src/exec/hdfs-parquet-scanner.cc ---------------------------------------------------------------------- diff --git a/be/src/exec/hdfs-parquet-scanner.cc b/be/src/exec/hdfs-parquet-scanner.cc index f20818a..4cd4340 100644 --- a/be/src/exec/hdfs-parquet-scanner.cc +++ b/be/src/exec/hdfs-parquet-scanner.cc @@ -556,7 +556,7 @@ Status HdfsParquetScanner::EvaluateStatsConjuncts( // comparisons cannot happen here, since predicates with NULL literals are filtered // in the frontend. *skip_row_group = true; - return Status::OK(); + break; } if (pos_field) { @@ -597,10 +597,13 @@ Status HdfsParquetScanner::EvaluateStatsConjuncts( row.SetTuple(0, min_max_tuple); if (!ExecNode::EvalPredicate(eval, &row)) { *skip_row_group = true; - return Status::OK(); + break; } } } + + // Free any local allocations accumulated during conjunct evaluation. + ScalarExprEvaluator::FreeLocalAllocations(min_max_conjunct_evals_); return Status::OK(); } @@ -911,6 +914,11 @@ Status HdfsParquetScanner::EvalDictionaryFilters(const parquet::RowGroup& row_gr void* slot = dict_filter_tuple->GetSlot(slot_desc->tuple_offset()); bool column_has_match = false; for (int dict_idx = 0; dict_idx < dictionary->num_entries(); ++dict_idx) { + if (dict_idx % 1024 == 0) { + // Don't let local allocations accumulate too much for large dictionaries or + // many row groups. + ScalarExprEvaluator::FreeLocalAllocations(dict_filter_conjunct_evals); + } dictionary->GetValue(dict_idx, slot); // We can only eliminate this row group if no value from the dictionary matches. @@ -923,6 +931,8 @@ Status HdfsParquetScanner::EvalDictionaryFilters(const parquet::RowGroup& row_gr break; } } + // Free all local allocations now that we're done with the filter. + ScalarExprEvaluator::FreeLocalAllocations(dict_filter_conjunct_evals); if (!column_has_match) { // The column contains no value that matches the conjunct. The row group
