This is an automated email from the ASF dual-hosted git repository.
cwylie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/master by this push:
new 48731710fb precursor changes for nested columns to minimize files
changed (#12714)
48731710fb is described below
commit 48731710fb54270758e29609968b6a1747616de6
Author: Clint Wylie <[email protected]>
AuthorDate: Fri Jul 1 02:27:19 2022 -0700
precursor changes for nested columns to minimize files changed (#12714)
* precursor changes for nested columns to minimize files changed
* inspection fix
* visibility
* adjustment
* unecessary change
---
.../segment/DictionaryEncodedColumnMerger.java | 2 +-
.../druid/segment/DictionaryMergingIterator.java | 5 ++
.../apache/druid/segment/column/ColumnBuilder.java | 5 ++
.../druid/segment/data/CompressionStrategy.java | 14 ++---
.../incremental/IncrementalIndexAdapter.java | 72 ++++++++++++----------
.../serde/DictionaryEncodedColumnPartSerde.java | 8 +--
.../virtual/ExpressionVectorInputBinding.java | 2 +-
.../query/aggregation/AggregationTestHelper.java | 5 ++
.../calcite/expression/OperatorConversions.java | 7 +++
.../expression/builtin/CastOperatorConversion.java | 11 +++-
.../sql/calcite/planner/DruidRexExecutor.java | 3 +-
.../sql/calcite/rel/VirtualColumnRegistry.java | 8 +++
12 files changed, 91 insertions(+), 51 deletions(-)
diff --git
a/processing/src/main/java/org/apache/druid/segment/DictionaryEncodedColumnMerger.java
b/processing/src/main/java/org/apache/druid/segment/DictionaryEncodedColumnMerger.java
index 84b35cedc2..e74050f741 100644
---
a/processing/src/main/java/org/apache/druid/segment/DictionaryEncodedColumnMerger.java
+++
b/processing/src/main/java/org/apache/druid/segment/DictionaryEncodedColumnMerger.java
@@ -187,7 +187,7 @@ public abstract class DictionaryEncodedColumnMerger<T
extends Comparable<T>> imp
dimConversions.set(i, dictionaryMergeIterator.conversions[i]);
}
}
- cardinality = dictionaryMergeIterator.counter;
+ cardinality = dictionaryMergeIterator.getCardinality();
} else if (numMergeIndex == 1) {
writeDictionary(dimValueLookup);
cardinality = dimValueLookup.size();
diff --git
a/processing/src/main/java/org/apache/druid/segment/DictionaryMergingIterator.java
b/processing/src/main/java/org/apache/druid/segment/DictionaryMergingIterator.java
index 56950d8780..2e570c7d3d 100644
---
a/processing/src/main/java/org/apache/druid/segment/DictionaryMergingIterator.java
+++
b/processing/src/main/java/org/apache/druid/segment/DictionaryMergingIterator.java
@@ -124,6 +124,11 @@ public class DictionaryMergingIterator<T extends
Comparable<T>> implements Close
return value;
}
+ public int getCardinality()
+ {
+ return counter;
+ }
+
protected PeekingIterator<T> transformIndexedIterator(Indexed<T> indexed)
{
return Iterators.peekingIterator(
diff --git
a/processing/src/main/java/org/apache/druid/segment/column/ColumnBuilder.java
b/processing/src/main/java/org/apache/druid/segment/column/ColumnBuilder.java
index 70f67bc13e..32fdbc607f 100644
---
a/processing/src/main/java/org/apache/druid/segment/column/ColumnBuilder.java
+++
b/processing/src/main/java/org/apache/druid/segment/column/ColumnBuilder.java
@@ -42,6 +42,11 @@ public class ColumnBuilder
@Nullable
private SmooshedFileMapper fileMapper = null;
+ @SuppressWarnings("unused")
+ public ColumnCapabilitiesImpl getCapabilitiesBuilder()
+ {
+ return capabilitiesBuilder;
+ }
public ColumnBuilder setFileMapper(SmooshedFileMapper fileMapper)
{
diff --git
a/processing/src/main/java/org/apache/druid/segment/data/CompressionStrategy.java
b/processing/src/main/java/org/apache/druid/segment/data/CompressionStrategy.java
index 478983f941..96550329a8 100644
---
a/processing/src/main/java/org/apache/druid/segment/data/CompressionStrategy.java
+++
b/processing/src/main/java/org/apache/druid/segment/data/CompressionStrategy.java
@@ -190,7 +190,7 @@ public enum CompressionStrategy
* <p>
* If the allocated buffer is a direct buffer, it should be registered to
be freed with the given Closer.
*/
- ByteBuffer allocateInBuffer(int inputSize, Closer closer)
+ public ByteBuffer allocateInBuffer(int inputSize, Closer closer)
{
return ByteBuffer.allocate(inputSize);
}
@@ -203,7 +203,7 @@ public enum CompressionStrategy
* <p>
* If the allocated buffer is a direct buffer, it should be registered to
be freed with the given Closer.
*/
- abstract ByteBuffer allocateOutBuffer(int inputSize, Closer closer);
+ public abstract ByteBuffer allocateOutBuffer(int inputSize, Closer closer);
/**
* Returns a ByteBuffer with compressed contents of in between it's
position and limit. It may be the provided out
@@ -221,7 +221,7 @@ public enum CompressionStrategy
private static final UncompressedCompressor DEFAULT_COMPRESSOR = new
UncompressedCompressor();
@Override
- ByteBuffer allocateOutBuffer(int inputSize, Closer closer)
+ public ByteBuffer allocateOutBuffer(int inputSize, Closer closer)
{
return ByteBuffer.allocate(inputSize);
}
@@ -333,7 +333,7 @@ public enum CompressionStrategy
}
@Override
- ByteBuffer allocateInBuffer(int inputSize, Closer closer)
+ public ByteBuffer allocateInBuffer(int inputSize, Closer closer)
{
ByteBuffer inBuffer = ByteBuffer.allocateDirect(inputSize);
closer.register(() -> ByteBufferUtils.free(inBuffer));
@@ -341,7 +341,7 @@ public enum CompressionStrategy
}
@Override
- ByteBuffer allocateOutBuffer(int inputSize, Closer closer)
+ public ByteBuffer allocateOutBuffer(int inputSize, Closer closer)
{
ByteBuffer outBuffer =
ByteBuffer.allocateDirect(LZ4_HIGH.maxCompressedLength(inputSize));
closer.register(() -> ByteBufferUtils.free(outBuffer));
@@ -365,7 +365,7 @@ public enum CompressionStrategy
private static final ZstdCompressor DEFAULT_COMPRESSOR = new
ZstdCompressor();
@Override
- ByteBuffer allocateInBuffer(int inputSize, Closer closer)
+ public ByteBuffer allocateInBuffer(int inputSize, Closer closer)
{
ByteBuffer inBuffer = ByteBuffer.allocateDirect(inputSize);
closer.register(() -> ByteBufferUtils.free(inBuffer));
@@ -373,7 +373,7 @@ public enum CompressionStrategy
}
@Override
- ByteBuffer allocateOutBuffer(int inputSize, Closer closer)
+ public ByteBuffer allocateOutBuffer(int inputSize, Closer closer)
{
ByteBuffer outBuffer = ByteBuffer.allocateDirect((int)
Zstd.compressBound(inputSize));
closer.register(() -> ByteBufferUtils.free(outBuffer));
diff --git
a/processing/src/main/java/org/apache/druid/segment/incremental/IncrementalIndexAdapter.java
b/processing/src/main/java/org/apache/druid/segment/incremental/IncrementalIndexAdapter.java
index 1f2be9ed1e..44f6bebd02 100644
---
a/processing/src/main/java/org/apache/druid/segment/incremental/IncrementalIndexAdapter.java
+++
b/processing/src/main/java/org/apache/druid/segment/incremental/IncrementalIndexAdapter.java
@@ -45,25 +45,6 @@ public class IncrementalIndexAdapter implements
IndexableAdapter
private final IncrementalIndex index;
private final Map<String, DimensionAccessor> accessors;
- private static class DimensionAccessor
- {
- private final IncrementalIndex.DimensionDesc dimensionDesc;
- @Nullable
- private final MutableBitmap[] invertedIndexes;
- private final DimensionIndexer indexer;
-
- public DimensionAccessor(IncrementalIndex.DimensionDesc dimensionDesc)
- {
- this.dimensionDesc = dimensionDesc;
- this.indexer = dimensionDesc.getIndexer();
- if (dimensionDesc.getCapabilities().hasBitmapIndexes()) {
- this.invertedIndexes = new MutableBitmap[indexer.getCardinality() + 1];
- } else {
- this.invertedIndexes = null;
- }
- }
- }
-
public IncrementalIndexAdapter(Interval dataInterval, IncrementalIndex
index, BitmapFactory bitmapFactory)
{
this.dataInterval = dataInterval;
@@ -118,6 +99,12 @@ public class IncrementalIndexAdapter implements
IndexableAdapter
}
}
+ @SuppressWarnings("unused")
+ public IncrementalIndex getIncrementalIndex()
+ {
+ return index;
+ }
+
@Override
public Interval getDataInterval()
{
@@ -190,6 +177,24 @@ public class IncrementalIndexAdapter implements
IndexableAdapter
return new MutableBitmapValues(bitmapIndex);
}
+ @Override
+ public String getMetricType(String metric)
+ {
+ return index.getMetricType(metric);
+ }
+
+ @Override
+ public ColumnCapabilities getCapabilities(String column)
+ {
+ return index.getColumnCapabilities(column);
+ }
+
+ @Override
+ public Metadata getMetadata()
+ {
+ return index.getMetadata();
+ }
+
static class MutableBitmapValues implements BitmapValues
{
private final MutableBitmap bitmapIndex;
@@ -212,21 +217,22 @@ public class IncrementalIndexAdapter implements
IndexableAdapter
}
}
- @Override
- public String getMetricType(String metric)
- {
- return index.getMetricType(metric);
- }
-
- @Override
- public ColumnCapabilities getCapabilities(String column)
+ private static class DimensionAccessor
{
- return index.getColumnCapabilities(column);
- }
+ private final IncrementalIndex.DimensionDesc dimensionDesc;
+ @Nullable
+ private final MutableBitmap[] invertedIndexes;
+ private final DimensionIndexer indexer;
- @Override
- public Metadata getMetadata()
- {
- return index.getMetadata();
+ public DimensionAccessor(IncrementalIndex.DimensionDesc dimensionDesc)
+ {
+ this.dimensionDesc = dimensionDesc;
+ this.indexer = dimensionDesc.getIndexer();
+ if (dimensionDesc.getCapabilities().hasBitmapIndexes()) {
+ this.invertedIndexes = new MutableBitmap[indexer.getCardinality() + 1];
+ } else {
+ this.invertedIndexes = null;
+ }
+ }
}
}
diff --git
a/processing/src/main/java/org/apache/druid/segment/serde/DictionaryEncodedColumnPartSerde.java
b/processing/src/main/java/org/apache/druid/segment/serde/DictionaryEncodedColumnPartSerde.java
index bf1e66df35..2358372426 100644
---
a/processing/src/main/java/org/apache/druid/segment/serde/DictionaryEncodedColumnPartSerde.java
+++
b/processing/src/main/java/org/apache/druid/segment/serde/DictionaryEncodedColumnPartSerde.java
@@ -56,10 +56,10 @@ import java.nio.channels.WritableByteChannel;
public class DictionaryEncodedColumnPartSerde implements ColumnPartSerde
{
- private static final int NO_FLAGS = 0;
- private static final int STARTING_FLAGS = Feature.NO_BITMAP_INDEX.getMask();
+ public static final int NO_FLAGS = 0;
+ public static final int STARTING_FLAGS = Feature.NO_BITMAP_INDEX.getMask();
- enum Feature
+ public enum Feature
{
MULTI_VALUE,
MULTI_VALUE_V3,
@@ -76,7 +76,7 @@ public class DictionaryEncodedColumnPartSerde implements
ColumnPartSerde
}
}
- enum VERSION
+ public enum VERSION
{
UNCOMPRESSED_SINGLE_VALUE, // 0x0
UNCOMPRESSED_MULTI_VALUE, // 0x1
diff --git
a/processing/src/main/java/org/apache/druid/segment/virtual/ExpressionVectorInputBinding.java
b/processing/src/main/java/org/apache/druid/segment/virtual/ExpressionVectorInputBinding.java
index 5820c537ea..9e08749c0b 100644
---
a/processing/src/main/java/org/apache/druid/segment/virtual/ExpressionVectorInputBinding.java
+++
b/processing/src/main/java/org/apache/druid/segment/virtual/ExpressionVectorInputBinding.java
@@ -30,7 +30,7 @@ import javax.annotation.Nullable;
import java.util.HashMap;
import java.util.Map;
-class ExpressionVectorInputBinding implements Expr.VectorInputBinding
+public class ExpressionVectorInputBinding implements Expr.VectorInputBinding
{
private final Map<String, VectorValueSelector> numeric;
private final Map<String, VectorObjectSelector> objects;
diff --git
a/processing/src/test/java/org/apache/druid/query/aggregation/AggregationTestHelper.java
b/processing/src/test/java/org/apache/druid/query/aggregation/AggregationTestHelper.java
index 37c31ec132..595bc08f61 100644
---
a/processing/src/test/java/org/apache/druid/query/aggregation/AggregationTestHelper.java
+++
b/processing/src/test/java/org/apache/druid/query/aggregation/AggregationTestHelper.java
@@ -847,6 +847,11 @@ public class AggregationTestHelper implements Closeable
return results;
}
+ public IndexIO getIndexIO()
+ {
+ return indexIO;
+ }
+
@Override
public void close() throws IOException
{
diff --git
a/sql/src/main/java/org/apache/druid/sql/calcite/expression/OperatorConversions.java
b/sql/src/main/java/org/apache/druid/sql/calcite/expression/OperatorConversions.java
index f02db13c45..0a24da42c5 100644
---
a/sql/src/main/java/org/apache/druid/sql/calcite/expression/OperatorConversions.java
+++
b/sql/src/main/java/org/apache/druid/sql/calcite/expression/OperatorConversions.java
@@ -460,6 +460,13 @@ public class OperatorConversions
return this;
}
+ @SuppressWarnings("unused")
+ public OperatorBuilder operandTypeInference(SqlOperandTypeInference
operandTypeInference)
+ {
+ this.operandTypeInference = operandTypeInference;
+ return this;
+ }
+
/**
* Creates a {@link SqlFunction} from this builder.
*/
diff --git
a/sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/CastOperatorConversion.java
b/sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/CastOperatorConversion.java
index 97e6f48782..9062a32d0b 100644
---
a/sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/CastOperatorConversion.java
+++
b/sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/CastOperatorConversion.java
@@ -116,9 +116,14 @@ public class CastOperatorConversion implements
SqlOperatorConversion
} else if (SqlTypeName.DATETIME_TYPES.contains(fromType) &&
SqlTypeName.CHAR_TYPES.contains(toType)) {
return castDateTimeToChar(plannerContext, operandExpression, fromType,
Calcites.getColumnTypeForRelDataType(rexNode.getType()));
} else {
- // Handle other casts.
- final ExprType fromExprType = EXPRESSION_TYPES.get(fromType);
- final ExprType toExprType = EXPRESSION_TYPES.get(toType);
+ // Handle other casts. If either type is ANY, use the other type
instead. If both are ANY, this means nulls
+ // downstream, Druid will try its best
+ final ExprType fromExprType = SqlTypeName.ANY.equals(fromType)
+ ? EXPRESSION_TYPES.get(toType)
+ : EXPRESSION_TYPES.get(fromType);
+ final ExprType toExprType = SqlTypeName.ANY.equals(toType)
+ ? EXPRESSION_TYPES.get(fromType)
+ : EXPRESSION_TYPES.get(toType);
if (fromExprType == null || toExprType == null) {
// We have no runtime type for these SQL types.
diff --git
a/sql/src/main/java/org/apache/druid/sql/calcite/planner/DruidRexExecutor.java
b/sql/src/main/java/org/apache/druid/sql/calcite/planner/DruidRexExecutor.java
index 28e8b66504..5e3f14595a 100644
---
a/sql/src/main/java/org/apache/druid/sql/calcite/planner/DruidRexExecutor.java
+++
b/sql/src/main/java/org/apache/druid/sql/calcite/planner/DruidRexExecutor.java
@@ -32,7 +32,6 @@ import org.apache.druid.math.expr.Parser;
import org.apache.druid.segment.column.RowSignature;
import org.apache.druid.sql.calcite.expression.DruidExpression;
import org.apache.druid.sql.calcite.expression.Expressions;
-import org.apache.druid.sql.calcite.table.RowSignatures;
import java.math.BigDecimal;
import java.util.Arrays;
@@ -168,7 +167,7 @@ public class DruidRexExecutor implements RexExecutor
} else {
literal =
rexBuilder.makeLiteral(Arrays.asList(exprResult.asArray()), constExp.getType(),
true);
}
- } else if (sqlTypeName == SqlTypeName.OTHER && constExp.getType()
instanceof RowSignatures.ComplexSqlType) {
+ } else if (sqlTypeName == SqlTypeName.OTHER) {
// complex constant is not reducible, so just leave it as an
expression
literal = constExp;
} else {
diff --git
a/sql/src/main/java/org/apache/druid/sql/calcite/rel/VirtualColumnRegistry.java
b/sql/src/main/java/org/apache/druid/sql/calcite/rel/VirtualColumnRegistry.java
index ec6f8c7ee3..b3464b9bd6 100644
---
a/sql/src/main/java/org/apache/druid/sql/calcite/rel/VirtualColumnRegistry.java
+++
b/sql/src/main/java/org/apache/druid/sql/calcite/rel/VirtualColumnRegistry.java
@@ -20,6 +20,7 @@
package org.apache.druid.sql.calcite.rel;
import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.sql.type.SqlTypeName;
import org.apache.druid.math.expr.ExprMacroTable;
import org.apache.druid.segment.VirtualColumn;
import org.apache.druid.segment.column.ColumnType;
@@ -127,6 +128,13 @@ public class VirtualColumnRegistry
RelDataType typeHint
)
{
+ if (typeHint.getSqlTypeName() == SqlTypeName.OTHER &&
expression.getDruidType() != null) {
+ // fall back to druid type if sql type isn't very helpful
+ return getOrCreateVirtualColumnForExpression(
+ expression,
+ expression.getDruidType()
+ );
+ }
return getOrCreateVirtualColumnForExpression(
expression,
Calcites.getColumnTypeForRelDataType(typeHint)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
