This is an automated email from the ASF dual-hosted git repository.
siddteotia pushed a commit to branch hotfix-0630
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/hotfix-0630 by this push:
new a4683b5 Fix double memory allocation during operator setup (#5619)
a4683b5 is described below
commit a4683b5667611d095bf24e45e1d014455e8a0362
Author: Sidd <[email protected]>
AuthorDate: Thu Jun 25 15:13:08 2020 -0700
Fix double memory allocation during operator setup (#5619)
Co-authored-by: Siddharth Teotia <[email protected]>
---
.../org/apache/pinot/core/operator/ProjectionOperator.java | 9 +++++----
.../apache/pinot/core/operator/blocks/MultiValueBlock.java | 10 ++++++++--
.../apache/pinot/core/operator/blocks/ProjectionBlock.java | 14 ++++++++------
.../pinot/core/operator/blocks/SingleValueBlock.java | 8 ++++++--
4 files changed, 27 insertions(+), 14 deletions(-)
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/operator/ProjectionOperator.java
b/pinot-core/src/main/java/org/apache/pinot/core/operator/ProjectionOperator.java
index 2abf311..00dc5c8 100644
---
a/pinot-core/src/main/java/org/apache/pinot/core/operator/ProjectionOperator.java
+++
b/pinot-core/src/main/java/org/apache/pinot/core/operator/ProjectionOperator.java
@@ -24,6 +24,7 @@ import org.apache.pinot.core.common.Block;
import org.apache.pinot.core.common.DataBlockCache;
import org.apache.pinot.core.common.DataFetcher;
import org.apache.pinot.core.common.DataSource;
+import org.apache.pinot.core.common.DataSourceMetadata;
import org.apache.pinot.core.operator.blocks.DocIdSetBlock;
import org.apache.pinot.core.operator.blocks.ProjectionBlock;
@@ -32,15 +33,15 @@ public class ProjectionOperator extends
BaseOperator<ProjectionBlock> {
private static final String OPERATOR_NAME = "ProjectionOperator";
private final Map<String, DataSource> _dataSourceMap;
- private final Map<String, Block> _dataBlockMap;
+ private final Map<String, DataSourceMetadata> _dataSourceMetadataMap;
private final BaseOperator<DocIdSetBlock> _docIdSetOperator;
private final DataBlockCache _dataBlockCache;
public ProjectionOperator(Map<String, DataSource> dataSourceMap,
BaseOperator<DocIdSetBlock> docIdSetOperator) {
_dataSourceMap = dataSourceMap;
- _dataBlockMap = new HashMap<>(dataSourceMap.size());
+ _dataSourceMetadataMap = new HashMap<>(dataSourceMap.size());
for (Map.Entry<String, DataSource> entry : dataSourceMap.entrySet()) {
- _dataBlockMap.put(entry.getKey(), entry.getValue().nextBlock());
+ _dataSourceMetadataMap.put(entry.getKey(),
entry.getValue().getDataSourceMetadata());
}
_docIdSetOperator = docIdSetOperator;
_dataBlockCache = new DataBlockCache(new DataFetcher(dataSourceMap));
@@ -62,7 +63,7 @@ public class ProjectionOperator extends
BaseOperator<ProjectionBlock> {
return null;
} else {
_dataBlockCache.initNewBlock(docIdSetBlock.getDocIdSet(),
docIdSetBlock.getSearchableLength());
- return new ProjectionBlock(_dataBlockMap, _dataBlockCache,
docIdSetBlock);
+ return new ProjectionBlock(_dataSourceMetadataMap, _dataBlockCache,
docIdSetBlock);
}
}
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/operator/blocks/MultiValueBlock.java
b/pinot-core/src/main/java/org/apache/pinot/core/operator/blocks/MultiValueBlock.java
index 825dcc8..b07dc18 100644
---
a/pinot-core/src/main/java/org/apache/pinot/core/operator/blocks/MultiValueBlock.java
+++
b/pinot-core/src/main/java/org/apache/pinot/core/operator/blocks/MultiValueBlock.java
@@ -31,12 +31,15 @@ import org.apache.pinot.spi.data.FieldSpec;
@SuppressWarnings("rawtypes")
public final class MultiValueBlock implements Block {
- private final BlockValSet _blockValSet;
+ private final SingleColumnMultiValueReader _reader;
+ private final FieldSpec.DataType _dataType;
private final BlockMetadata _blockMetadata;
+ private BlockValSet _blockValSet;
public MultiValueBlock(SingleColumnMultiValueReader reader, int numDocs, int
maxNumMultiValues,
FieldSpec.DataType dataType, Dictionary dictionary) {
- _blockValSet = new MultiValueSet(reader, dataType);
+ _reader = reader;
+ _dataType = dataType;
_blockMetadata = new BlockMetadataImpl(numDocs, false, maxNumMultiValues,
dataType, dictionary);
}
@@ -47,6 +50,9 @@ public final class MultiValueBlock implements Block {
@Override
public BlockValSet getBlockValueSet() {
+ if (_blockValSet == null) {
+ _blockValSet = new MultiValueSet(_reader, _dataType);
+ }
return _blockValSet;
}
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/operator/blocks/ProjectionBlock.java
b/pinot-core/src/main/java/org/apache/pinot/core/operator/blocks/ProjectionBlock.java
index 1179885..c3b8774 100644
---
a/pinot-core/src/main/java/org/apache/pinot/core/operator/blocks/ProjectionBlock.java
+++
b/pinot-core/src/main/java/org/apache/pinot/core/operator/blocks/ProjectionBlock.java
@@ -25,7 +25,9 @@ import org.apache.pinot.core.common.BlockDocIdValueSet;
import org.apache.pinot.core.common.BlockMetadata;
import org.apache.pinot.core.common.BlockValSet;
import org.apache.pinot.core.common.DataBlockCache;
+import org.apache.pinot.core.common.DataSourceMetadata;
import org.apache.pinot.core.operator.docvalsets.ProjectionBlockValSet;
+import org.apache.pinot.spi.data.FieldSpec;
/**
@@ -33,12 +35,12 @@ import
org.apache.pinot.core.operator.docvalsets.ProjectionBlockValSet;
* It provides DocIdSetBlock for a given column.
*/
public class ProjectionBlock implements Block {
- private final Map<String, Block> _blockMap;
+ private final Map<String, DataSourceMetadata> _dataSourceMetadataMap;
private final DocIdSetBlock _docIdSetBlock;
private final DataBlockCache _dataBlockCache;
- public ProjectionBlock(Map<String, Block> blockMap, DataBlockCache
dataBlockCache, DocIdSetBlock docIdSetBlock) {
- _blockMap = blockMap;
+ public ProjectionBlock(Map<String, DataSourceMetadata>
dataSourceMetadataMap, DataBlockCache dataBlockCache, DocIdSetBlock
docIdSetBlock) {
+ _dataSourceMetadataMap = dataSourceMetadataMap;
_docIdSetBlock = docIdSetBlock;
_dataBlockCache = dataBlockCache;
}
@@ -64,9 +66,9 @@ public class ProjectionBlock implements Block {
}
public BlockValSet getBlockValueSet(String column) {
- BlockMetadata blockMetadata = _blockMap.get(column).getMetadata();
- return new ProjectionBlockValSet(_dataBlockCache, column,
blockMetadata.getDataType(),
- blockMetadata.isSingleValue());
+ FieldSpec fieldSpec = _dataSourceMetadataMap.get(column).getFieldSpec();
+ return new ProjectionBlockValSet(_dataBlockCache, column,
fieldSpec.getDataType(),
+ fieldSpec.isSingleValueField());
}
public DocIdSetBlock getDocIdSetBlock() {
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/operator/blocks/SingleValueBlock.java
b/pinot-core/src/main/java/org/apache/pinot/core/operator/blocks/SingleValueBlock.java
index 132e54e..6e9615b 100644
---
a/pinot-core/src/main/java/org/apache/pinot/core/operator/blocks/SingleValueBlock.java
+++
b/pinot-core/src/main/java/org/apache/pinot/core/operator/blocks/SingleValueBlock.java
@@ -32,13 +32,14 @@ import org.apache.pinot.spi.data.FieldSpec;
@SuppressWarnings("rawtypes")
public final class SingleValueBlock implements Block {
private final SingleColumnSingleValueReader _reader;
- private final BlockValSet _blockValSet;
+ private final FieldSpec.DataType _dataType;
private final BlockMetadata _blockMetadata;
+ private BlockValSet _blockValSet;
public SingleValueBlock(SingleColumnSingleValueReader reader, int numDocs,
FieldSpec.DataType dataType,
Dictionary dictionary) {
_reader = reader;
- _blockValSet = new SingleValueSet(reader, dataType);
+ _dataType = dataType;
_blockMetadata = new BlockMetadataImpl(numDocs, true, 0, dataType,
dictionary);
}
@@ -53,6 +54,9 @@ public final class SingleValueBlock implements Block {
@Override
public BlockValSet getBlockValueSet() {
+ if (_blockValSet == null) {
+ _blockValSet = new SingleValueSet(_reader, _dataType);
+ }
return _blockValSet;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
