IMPALA-3952: Clear scratch batch mem pool if Open() failed. We used to be able to hit a DCHECK in HdfsParquetScanner::Close() when using the legacy aggs/joins if HdfsParquetScanner::Open() failed. With the legacy aggs/joins the tuple ptrs of the scratch batch are allocated from the scratch batch's mem pool, and if Open() failed we never freed or transferred the scratch batch's mem pool.
Testing: I tested this patch together with the fix for IMPALA-3964 on core/hdfs with the legacy aggs and joins enabled. Change-Id: I55f32ed698a5b6fed8c28af1391aa07e1560e782 Reviewed-on: http://gerrit.cloudera.org:8080/3953 Reviewed-by: Alex Behm <[email protected]> Tested-by: Internal Jenkins Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/3a630a5d Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/3a630a5d Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/3a630a5d Branch: refs/heads/master Commit: 3a630a5d6871db6afe9ca343aea80b14c6525f3d Parents: 5a55ba7 Author: Alex Behm <[email protected]> Authored: Thu Aug 11 21:30:01 2016 -0700 Committer: Internal Jenkins <[email protected]> Committed: Tue Aug 16 06:08:50 2016 +0000 ---------------------------------------------------------------------- be/src/exec/hdfs-parquet-scanner.cc | 7 +++++++ 1 file changed, 7 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/3a630a5d/be/src/exec/hdfs-parquet-scanner.cc ---------------------------------------------------------------------- diff --git a/be/src/exec/hdfs-parquet-scanner.cc b/be/src/exec/hdfs-parquet-scanner.cc index b0fd008..1431689 100644 --- a/be/src/exec/hdfs-parquet-scanner.cc +++ b/be/src/exec/hdfs-parquet-scanner.cc @@ -50,6 +50,8 @@ using namespace impala; DEFINE_double(parquet_min_filter_reject_ratio, 0.1, "(Advanced) If the percentage of " "rows rejected by a runtime filter drops below this value, the filter is disabled."); +DECLARE_bool(enable_partitioned_aggregation); +DECLARE_bool(enable_partitioned_hash_join); // The number of rows between checks to see if a filter is not effective, and should be // disabled. Must be a power of two. @@ -205,6 +207,11 @@ void HdfsParquetScanner::Close(RowBatch* row_batch) { if (row_batch != NULL) { FlushRowGroupResources(row_batch); if (add_batches_to_queue_) scan_node_->AddMaterializedRowBatch(row_batch); + } else if (!FLAGS_enable_partitioned_hash_join || + !FLAGS_enable_partitioned_aggregation) { + // With the legacy aggs/joins the tuple ptrs of the scratch batch are allocated + // from the scratch batch's mem pool. We can get into this case if Open() fails. + scratch_batch_->mem_pool()->FreeAll(); } // Verify all resources (if any) have been transferred. DCHECK_EQ(dictionary_pool_.get()->total_allocated_bytes(), 0);
