This is an automated email from the ASF dual-hosted git repository. amashenkov pushed a commit to branch ignite-22703 in repository https://gitbox.apache.org/repos/asf/ignite-3.git
commit e66feba3f5f1563a0b0282d83ade355f5cd5d419 Author: amashenkov <amashen...@apache.org> AuthorDate: Wed Jul 2 21:56:04 2025 +0300 Pass projection to ScannableTable methods instead of bitset --- .../internal/benchmark/SqlSelectAllBenchmark.java | 154 +++++++++++++++++++++ .../engine/exec/ExecutableTableRegistryImpl.java | 5 +- .../sql/engine/exec/LogicalRelImplementor.java | 27 +++- .../exec/ProjectedTableRowConverterImpl.java | 18 +-- .../internal/sql/engine/exec/RowHandler.java | 5 + .../internal/sql/engine/exec/ScannableTable.java | 7 +- .../sql/engine/exec/ScannableTableImpl.java | 11 +- .../sql/engine/exec/TableRowConverterFactory.java | 22 ++- .../engine/exec/TableRowConverterFactoryImpl.java | 38 ++--- .../sql/engine/exec/rel/IndexScanNode.java | 5 +- .../sql/engine/exec/rel/TableScanNode.java | 6 +- .../engine/schema/AbstractIgniteDataSource.java | 10 +- .../sql/engine/schema/IgniteDataSource.java | 16 ++- .../sql/engine/schema/IgniteTableImpl.java | 44 ++++-- .../sql/engine/schema/TableDescriptor.java | 4 +- .../sql/engine/schema/TableDescriptorImpl.java | 6 +- .../internal/sql/engine/util/RowTypeUtils.java | 34 +++-- .../exec/ExecutableTableRegistrySelfTest.java | 1 - .../engine/exec/NoOpExecutableTableRegistry.java | 7 +- .../exec/ProjectedTableRowConverterSelfTest.java | 2 +- .../internal/sql/engine/exec/QueryTimeoutTest.java | 7 +- .../sql/engine/exec/TableRowConverterSelfTest.java | 3 +- .../exec/rel/IndexScanNodeExecutionTest.java | 7 +- .../engine/exec/rel/ScannableTableSelfTest.java | 24 ++-- .../sql/engine/framework/TestBuilders.java | 36 +++-- .../sql/engine/framework/TestClusterTest.java | 7 +- .../sql/engine/planner/AbstractPlannerTest.java | 3 +- .../sql/engine/planner/PlannerTimeoutTest.java | 6 +- .../sql/engine/prepare/PrepareServiceImplTest.java | 5 +- 29 files changed, 374 insertions(+), 146 deletions(-) diff --git a/modules/runner/src/integrationTest/java/org/apache/ignite/internal/benchmark/SqlSelectAllBenchmark.java b/modules/runner/src/integrationTest/java/org/apache/ignite/internal/benchmark/SqlSelectAllBenchmark.java new file mode 100644 index 00000000000..1b62824dd32 --- /dev/null +++ b/modules/runner/src/integrationTest/java/org/apache/ignite/internal/benchmark/SqlSelectAllBenchmark.java @@ -0,0 +1,154 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.benchmark; + +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.concurrent.TimeUnit; +import org.apache.ignite.internal.sql.engine.util.TpcTable; +import org.apache.ignite.internal.sql.engine.util.tpch.TpchTables; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Threads; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.infra.Blackhole; +import org.openjdk.jmh.runner.Runner; +import org.openjdk.jmh.runner.RunnerException; +import org.openjdk.jmh.runner.options.Options; +import org.openjdk.jmh.runner.options.OptionsBuilder; + +/** + * Benchmark that runs sql queries from TPC-H suite via embedded client. + */ +@State(Scope.Benchmark) +@Fork(1) +@Threads(1) +@Warmup(iterations = 10, time = 2) +@Measurement(iterations = 20, time = 2) +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.MILLISECONDS) +@SuppressWarnings({"WeakerAccess", "unused"}) +public class SqlSelectAllBenchmark extends AbstractTpcBenchmark { + /* + Minimal configuration of this benchmark requires specifying pathToDataset. Latest known location + of dataset is https://github.com/cmu-db/benchbase/tree/main/data/tpch-sf0.01 for scale factor 0.01 + and https://github.com/cmu-db/benchbase/tree/main/data/tpch-sf0.1 for scale factor 0.1. Dataset + is set of CSV files with name `{$tableName}.tbl` per each table and character `|` as separator. + + By default, cluster's work directory will be created as a temporary folder. This implies, + that all data generated by benchmark will be cleared automatically. However, this also implies + that cluster will be recreated on EVERY RUN. Given there are 25 queries, it results in 25 schema + initialization and data upload cycles. To initialize cluster once and then reuse it state override + `AbstractMultiNodeBenchmark.workDir()` method. Don't forget to clear that directory afterwards. + */ + + @Override + TpcTable[] tablesToInit() { + return new TpcTable[] { TpchTables.LINEITEM }; + } + + @Override + protected Path workDir() throws Exception { + return Paths.get("/Users/amashenkov/w/ignite-3/modules/runner/work/"); + } + + @Override + protected int nodes() { + return 1; + } + + @Override + Path pathToDataset() { + return Paths.get("/Users/amashenkov/w/datasets/tpc-h/sf-0.1"); + } + + @Param({ + "IDENTITY", "RESHUFFLING" + }) + private Projection projection; + + /** Benchmark that measures performance of queries from TPC-H suite. */ + @Benchmark + public void run(Blackhole bh) { + try (var rs = sql.execute(null, projection.query())) { + while (rs.hasNext()) { + bh.consume(rs.next()); + } + } + } + + /** + * Benchmark's entry point. + */ + public static void main(String[] args) throws RunnerException { + Options opt = new OptionsBuilder() + .include(".*\\." + SqlSelectAllBenchmark.class.getSimpleName() + ".*") + .build(); + + new Runner(opt).run(); + } + + public enum Projection { + IDENTITY { + @Override + public String query() { + StringBuilder sb = new StringBuilder("SELECT ") + .append(TpchTables.LINEITEM.columnName(0)); + + for (int i = 1; i < TpchTables.LINEITEM.columnsCount(); i++) { + sb.append(", ").append(TpchTables.LINEITEM.columnName(i)); + } + return sb.append(" FROM lineitem").toString(); + } + }, + TRIMMING { + @Override + public String query() { + StringBuilder sb = new StringBuilder("SELECT ") + .append(TpchTables.LINEITEM.columnName(0)); + + for (int i = 2; i < TpchTables.LINEITEM.columnsCount(); i++) { + sb.append(", ").append(TpchTables.LINEITEM.columnName(i)); + } + return sb.append(" FROM lineitem").toString(); + } + }, + RESHUFFLING { + @Override + public String query() { + StringBuilder sb = new StringBuilder("SELECT ") + .append(TpchTables.LINEITEM.columnName(1)) + .append(", ").append(TpchTables.LINEITEM.columnName(0)); + + for (int i = 2; i < TpchTables.LINEITEM.columnsCount(); i++) { + sb.append(", ").append(TpchTables.LINEITEM.columnName(i)); + } + return sb.append(" FROM lineitem").toString(); + } + }; + + abstract String query(); + } +} diff --git a/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/exec/ExecutableTableRegistryImpl.java b/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/exec/ExecutableTableRegistryImpl.java index da19f76b94e..fe4b44bb2e3 100644 --- a/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/exec/ExecutableTableRegistryImpl.java +++ b/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/exec/ExecutableTableRegistryImpl.java @@ -17,8 +17,10 @@ package org.apache.ignite.internal.sql.engine.exec; +import java.util.BitSet; import java.util.Objects; import java.util.function.Supplier; +import org.apache.calcite.util.ImmutableIntList; import org.apache.ignite.internal.components.NodeProperties; import org.apache.ignite.internal.hlc.ClockService; import org.apache.ignite.internal.replicator.ReplicaService; @@ -99,7 +101,8 @@ public class ExecutableTableRegistryImpl implements ExecutableTableRegistry { InternalTable internalTable = table.internalTable(); ScannableTable scannableTable = new ScannableTableImpl(internalTable, converterFactory); - TableRowConverter rowConverter = converterFactory.create(null); + // TODO remove cast + TableRowConverter rowConverter = converterFactory.create((ImmutableIntList) null); UpdatableTableImpl updatableTable = new UpdatableTableImpl( sqlTable.id(), diff --git a/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/exec/LogicalRelImplementor.java b/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/exec/LogicalRelImplementor.java index 81556d31dc9..df3490052ca 100644 --- a/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/exec/LogicalRelImplementor.java +++ b/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/exec/LogicalRelImplementor.java @@ -443,7 +443,9 @@ public class LogicalRelImplementor<RowT> implements IgniteRelVisitor<Node<RowT>> IgniteTypeFactory typeFactory = ctx.getTypeFactory(); ImmutableBitSet requiredColumns = rel.requiredColumns(); - RelDataType rowType = tbl.getRowType(typeFactory, requiredColumns); + + ImmutableIntList requiredColumns0 = requiredColumns == null ? null : ImmutableIntList.copyOf(requiredColumns.asList()); + RelDataType rowType = tbl.getRowType(typeFactory, requiredColumns0); ScannableTable scannableTable = resolvedDependencies.scannableTable(tbl.id()); IgniteIndex idx = tbl.indexes().get(rel.indexName()); @@ -529,7 +531,7 @@ public class LogicalRelImplementor<RowT> implements IgniteRelVisitor<Node<RowT>> ranges, filters, prj, - requiredColumns == null ? null : requiredColumns.toBitSet() + requiredColumns0 ); } @@ -545,13 +547,27 @@ public class LogicalRelImplementor<RowT> implements IgniteRelVisitor<Node<RowT>> IgniteTypeFactory typeFactory = ctx.getTypeFactory(); - RelDataType rowType = tbl.getRowType(typeFactory, requiredColumns); + ImmutableIntList requiredColumns0 = requiredColumns == null ? null : ImmutableIntList.copyOf(requiredColumns.asList()); + RelDataType rowType = tbl.getRowType(typeFactory, requiredColumns0); Predicate<RowT> filters = null; if (condition != null) { SqlPredicate<RowT> sqlPredicate = expressionFactory.predicate(condition, rowType); filters = row -> sqlPredicate.test(ctx, row); } +// +// ImmutableIntList requiredColumnsMapping; +// if (projects != null) { +// int[] columnIndexes = new int[projects.size()]; +// for (int i = 0; i < projects.size(); i++) { +// columnIndexes[i] = ((RexLocalRef) projects.get(i)).getIndex(); +// } +// requiredColumnsMapping = ImmutableIntList.of(columnIndexes); +// +// projects = null; +// } else { +// requiredColumnsMapping = requiredColumns0; +// } Function<RowT, RowT> prj = null; if (projects != null) { @@ -581,7 +597,7 @@ public class LogicalRelImplementor<RowT> implements IgniteRelVisitor<Node<RowT>> partitionProvider, filters, prj, - requiredColumns == null ? null : requiredColumns.toBitSet() + requiredColumns0 ); } @@ -599,7 +615,8 @@ public class LogicalRelImplementor<RowT> implements IgniteRelVisitor<Node<RowT>> IgniteTypeFactory typeFactory = ctx.getTypeFactory(); - RelDataType rowType = igniteDataSource.getRowType(typeFactory, requiredColumns); + ImmutableIntList requiredColumns0 = requiredColumns == null ? null : ImmutableIntList.copyOf(requiredColumns.asList()); + RelDataType rowType = igniteDataSource.getRowType(typeFactory, requiredColumns0); Predicate<RowT> filters = null; if (condition != null) { diff --git a/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/exec/ProjectedTableRowConverterImpl.java b/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/exec/ProjectedTableRowConverterImpl.java index d0b99f578a2..9c29f221ec5 100644 --- a/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/exec/ProjectedTableRowConverterImpl.java +++ b/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/exec/ProjectedTableRowConverterImpl.java @@ -45,27 +45,13 @@ public class ProjectedTableRowConverterImpl extends TableRowConverterImpl { ProjectedTableRowConverterImpl( SchemaRegistry schemaRegistry, SchemaDescriptor schemaDescriptor, - BitSet requiredColumns, + int[] requiredColumns, Int2ObjectMap<VirtualColumn> extraColumns ) { super(schemaRegistry, schemaDescriptor); + this.requiredColumnsMapping = requiredColumns; this.virtualColumns = extraColumns; - - int size = requiredColumns.cardinality(); - - requiredColumnsMapping = new int[size]; - - int requiredIndex = 0; - for (Column column : schemaDescriptor.columns()) { - if (requiredColumns.get(column.positionInRow())) { - requiredColumnsMapping[requiredIndex++] = column.positionInRow(); - } - } - - for (VirtualColumn col : extraColumns.values()) { - requiredColumnsMapping[requiredIndex++] = col.columnIndex(); - } } @Override diff --git a/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/exec/RowHandler.java b/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/exec/RowHandler.java index 08c00dc89ae..e0dee713cc6 100644 --- a/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/exec/RowHandler.java +++ b/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/exec/RowHandler.java @@ -17,6 +17,7 @@ package org.apache.ignite.internal.sql.engine.exec; +import java.nio.ByteBuffer; import org.apache.ignite.internal.lang.InternalTuple; import org.apache.ignite.internal.schema.BinaryTuple; import org.apache.ignite.internal.sql.engine.exec.row.RowSchema; @@ -54,6 +55,10 @@ public interface RowHandler<RowT> { */ BinaryTuple toBinaryTuple(RowT row); + default ByteBuffer toByteBuffer(RowT row) { + return toBinaryTuple(row).byteBuffer(); + } + /** String representation. */ String toString(RowT row); diff --git a/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/exec/ScannableTable.java b/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/exec/ScannableTable.java index 29e2427e463..5aa373266f7 100644 --- a/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/exec/ScannableTable.java +++ b/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/exec/ScannableTable.java @@ -21,6 +21,7 @@ import java.util.BitSet; import java.util.List; import java.util.concurrent.CompletableFuture; import java.util.concurrent.Flow.Publisher; +import org.apache.calcite.util.ImmutableIntList; import org.apache.ignite.internal.sql.engine.exec.RowHandler.RowFactory; import org.apache.ignite.internal.sql.engine.exec.exp.RangeCondition; import org.apache.ignite.internal.tx.InternalTransaction; @@ -45,7 +46,7 @@ public interface ScannableTable { ExecutionContext<RowT> ctx, PartitionWithConsistencyToken partWithConsistencyToken, RowFactory<RowT> rowFactory, - @Nullable BitSet requiredColumns + @Nullable ImmutableIntList requiredColumns ); /** @@ -68,7 +69,7 @@ public interface ScannableTable { int indexId, List<String> columns, @Nullable RangeCondition<RowT> cond, - @Nullable BitSet requiredColumns + @Nullable ImmutableIntList requiredColumns ); /** @@ -91,7 +92,7 @@ public interface ScannableTable { int indexId, List<String> columns, RowT key, - @Nullable BitSet requiredColumns + @Nullable ImmutableIntList requiredColumns ); /** diff --git a/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/exec/ScannableTableImpl.java b/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/exec/ScannableTableImpl.java index c2d3677f412..b50aa1afece 100644 --- a/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/exec/ScannableTableImpl.java +++ b/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/exec/ScannableTableImpl.java @@ -25,6 +25,7 @@ import java.util.BitSet; import java.util.List; import java.util.concurrent.CompletableFuture; import java.util.concurrent.Flow.Publisher; +import org.apache.calcite.util.ImmutableIntList; import org.apache.ignite.internal.hlc.HybridTimestamp; import org.apache.ignite.internal.schema.BinaryRow; import org.apache.ignite.internal.schema.BinaryRowEx; @@ -56,7 +57,7 @@ public class ScannableTableImpl implements ScannableTable { /** {@inheritDoc} */ @Override public <RowT> Publisher<RowT> scan(ExecutionContext<RowT> ctx, PartitionWithConsistencyToken partWithConsistencyToken, - RowFactory<RowT> rowFactory, @Nullable BitSet requiredColumns) { + RowFactory<RowT> rowFactory, ImmutableIntList requiredColumns) { Publisher<BinaryRow> pub; TxAttributes txAttributes = ctx.txAttributes(); @@ -101,7 +102,7 @@ public class ScannableTableImpl implements ScannableTable { int indexId, List<String> columns, @Nullable RangeCondition<RowT> cond, - @Nullable BitSet requiredColumns + @Nullable ImmutableIntList requiredColumns ) { TxAttributes txAttributes = ctx.txAttributes(); RowHandler<RowT> handler = rowFactory.handler(); @@ -140,7 +141,7 @@ public class ScannableTableImpl implements ScannableTable { lower, upper, flags, - requiredColumns, + null, txAttributes.coordinatorId() ); } else { @@ -154,7 +155,7 @@ public class ScannableTableImpl implements ScannableTable { lower, upper, flags, - requiredColumns + null ); } @@ -172,7 +173,7 @@ public class ScannableTableImpl implements ScannableTable { int indexId, List<String> columns, RowT key, - @Nullable BitSet requiredColumns + @Nullable ImmutableIntList requiredColumns ) { TxAttributes txAttributes = ctx.txAttributes(); RowHandler<RowT> handler = rowFactory.handler(); diff --git a/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/exec/TableRowConverterFactory.java b/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/exec/TableRowConverterFactory.java index c8ff3f8ad9c..ed2bc762714 100644 --- a/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/exec/TableRowConverterFactory.java +++ b/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/exec/TableRowConverterFactory.java @@ -18,6 +18,8 @@ package org.apache.ignite.internal.sql.engine.exec; import java.util.BitSet; +import org.apache.calcite.util.ImmutableBitSet; +import org.apache.calcite.util.ImmutableIntList; import org.jetbrains.annotations.Nullable; /** @@ -26,9 +28,23 @@ import org.jetbrains.annotations.Nullable; */ @FunctionalInterface public interface TableRowConverterFactory { - TableRowConverter create(@Nullable BitSet requiredColumns); + TableRowConverter create(@Nullable ImmutableIntList projection); - default TableRowConverter create(BitSet requiredColumns, int partId) { - return create(requiredColumns); + default TableRowConverter create(@Nullable BitSet requiredColumns) { + if (requiredColumns == null) { + return create((ImmutableIntList) null); + } + return create(ImmutableIntList.copyOf(ImmutableBitSet.fromBitSet(requiredColumns).asList())); + } + + default TableRowConverter create(@Nullable BitSet requiredColumns, int partId) { + if (requiredColumns == null) { + return create((ImmutableIntList) null, partId); + } + return create(ImmutableIntList.copyOf(ImmutableBitSet.fromBitSet(requiredColumns).asList()), partId); + } + + default TableRowConverter create(@Nullable ImmutableIntList projection, int partId) { + return create(projection); } } diff --git a/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/exec/TableRowConverterFactoryImpl.java b/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/exec/TableRowConverterFactoryImpl.java index a7425a02762..c7f269b7e5d 100644 --- a/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/exec/TableRowConverterFactoryImpl.java +++ b/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/exec/TableRowConverterFactoryImpl.java @@ -22,6 +22,8 @@ import it.unimi.dsi.fastutil.ints.Int2ObjectMap; import it.unimi.dsi.fastutil.ints.Int2ObjectMaps; import java.util.BitSet; import java.util.function.IntFunction; +import java.util.stream.IntStream; +import org.apache.calcite.util.ImmutableIntList; import org.apache.ignite.internal.schema.SchemaDescriptor; import org.apache.ignite.internal.schema.SchemaRegistry; import org.apache.ignite.internal.sql.engine.schema.ColumnDescriptor; @@ -37,7 +39,7 @@ public class TableRowConverterFactoryImpl implements TableRowConverterFactory { private final SchemaRegistry schemaRegistry; private final SchemaDescriptor schemaDescriptor; private final TableRowConverter fullRowConverter; - private final BitSet tableColumnSet; + private final int[] tableColumnSet; private final Int2ObjectArrayMap<IntFunction<VirtualColumn>> virtualColumnsFactory = new Int2ObjectArrayMap<>(); /** @@ -62,8 +64,7 @@ public class TableRowConverterFactoryImpl implements TableRowConverterFactory { schemaDescriptor ); - tableColumnSet = new BitSet(); - tableColumnSet.set(0, tableDescriptor.columnsCount()); + tableColumnSet = IntStream.range(0, tableDescriptor.columnsCount()).toArray(); addVirtualColumn(tableDescriptor.columnDescriptor(Commons.PART_COL_NAME)); addVirtualColumn(tableDescriptor.columnDescriptor(Commons.PART_COL_NAME_LEGACY)); @@ -82,32 +83,35 @@ public class TableRowConverterFactoryImpl implements TableRowConverterFactory { } @Override - public TableRowConverter create(@Nullable BitSet requiredColumns) { - // TODO: IGNITE-22823 fix this. UpdatableTable must pass the bitset with updatable columns. - if (requiredColumns == null) { + public TableRowConverter create(@Nullable ImmutableIntList projection) { + // TODO: IGNITE-22823 fix this. UpdatableTable must pass the project with updatable columns. + if (projection == null) { return fullRowConverter; } - return create(requiredColumns, -1); + return create(projection, -1); } @Override - public TableRowConverter create(@Nullable BitSet requiredColumns, int partId) { - if (requiredColumns == null) { - requiredColumns = tableColumnSet; - } + public TableRowConverter create(@Nullable ImmutableIntList projection, int partId) { + int[] mapping = projection == null + ? tableColumnSet + : projection.toIntArray(); - boolean requireVirtualColumn = requiredColumns.nextSetBit(schemaDescriptor.length()) != -1; - if (!requireVirtualColumn && requiredColumns.cardinality() == schemaDescriptor.length()) { - return fullRowConverter; - } + // TODO: proper implementation +// boolean requireVirtualColumn = IntStream.of(mapping).anyMatch(i -> i >= schemaDescriptor.length()); +// Int2ObjectMap<VirtualColumn> extraColumns = requireVirtualColumn +// ? createVirtualColumns(requiredColumns, partId) +// : Int2ObjectMaps.emptyMap(); + + Int2ObjectMap<VirtualColumn> extraColumns = Int2ObjectMaps.emptyMap(); return new ProjectedTableRowConverterImpl( schemaRegistry, schemaDescriptor, - requiredColumns, - requireVirtualColumn ? createVirtualColumns(requiredColumns, partId) : Int2ObjectMaps.emptyMap() + mapping, + extraColumns ); } diff --git a/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/exec/rel/IndexScanNode.java b/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/exec/rel/IndexScanNode.java index 987b8fdf9d4..1e90c19d466 100644 --- a/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/exec/rel/IndexScanNode.java +++ b/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/exec/rel/IndexScanNode.java @@ -27,6 +27,7 @@ import java.util.function.Function; import java.util.function.Predicate; import java.util.stream.Collectors; import org.apache.calcite.rel.RelFieldCollation; +import org.apache.calcite.util.ImmutableIntList; import org.apache.ignite.internal.sql.engine.exec.ExecutionContext; import org.apache.ignite.internal.sql.engine.exec.PartitionProvider; import org.apache.ignite.internal.sql.engine.exec.PartitionWithConsistencyToken; @@ -57,7 +58,7 @@ public class IndexScanNode<RowT> extends StorageScanNode<RowT> { private final PartitionProvider<RowT> partitionProvider; /** Participating columns. */ - private final @Nullable BitSet requiredColumns; + private final @Nullable ImmutableIntList requiredColumns; private final @Nullable RangeIterable<RowT> rangeConditions; @@ -89,7 +90,7 @@ public class IndexScanNode<RowT> extends StorageScanNode<RowT> { @Nullable RangeIterable<RowT> rangeConditions, @Nullable Predicate<RowT> filters, @Nullable Function<RowT, RowT> rowTransformer, - @Nullable BitSet requiredColumns + @Nullable ImmutableIntList requiredColumns ) { super(ctx, filters, rowTransformer); diff --git a/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/exec/rel/TableScanNode.java b/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/exec/rel/TableScanNode.java index 5425978b952..c348f3d919f 100644 --- a/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/exec/rel/TableScanNode.java +++ b/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/exec/rel/TableScanNode.java @@ -17,12 +17,12 @@ package org.apache.ignite.internal.sql.engine.exec.rel; -import java.util.BitSet; import java.util.Iterator; import java.util.List; import java.util.concurrent.Flow.Publisher; import java.util.function.Function; import java.util.function.Predicate; +import org.apache.calcite.util.ImmutableIntList; import org.apache.ignite.internal.sql.engine.exec.ExecutionContext; import org.apache.ignite.internal.sql.engine.exec.PartitionProvider; import org.apache.ignite.internal.sql.engine.exec.PartitionWithConsistencyToken; @@ -46,7 +46,7 @@ public class TableScanNode<RowT> extends StorageScanNode<RowT> { private final RowFactory<RowT> rowFactory; - private final @Nullable BitSet requiredColumns; + private final @Nullable ImmutableIntList requiredColumns; /** * Constructor. @@ -67,7 +67,7 @@ public class TableScanNode<RowT> extends StorageScanNode<RowT> { PartitionProvider<RowT> partitionProvider, @Nullable Predicate<RowT> filters, @Nullable Function<RowT, RowT> rowTransformer, - @Nullable BitSet requiredColumns + @Nullable ImmutableIntList requiredColumns ) { super(ctx, filters, rowTransformer); diff --git a/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/schema/AbstractIgniteDataSource.java b/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/schema/AbstractIgniteDataSource.java index 28856e0812c..d99c01e2c0d 100644 --- a/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/schema/AbstractIgniteDataSource.java +++ b/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/schema/AbstractIgniteDataSource.java @@ -31,7 +31,7 @@ import org.apache.calcite.schema.Statistic; import org.apache.calcite.schema.impl.AbstractTable; import org.apache.calcite.sql.SqlCall; import org.apache.calcite.sql.SqlNode; -import org.apache.calcite.util.ImmutableBitSet; +import org.apache.calcite.util.ImmutableIntList; import org.apache.ignite.internal.sql.engine.trait.IgniteDistribution; import org.apache.ignite.internal.sql.engine.type.IgniteTypeFactory; import org.jetbrains.annotations.Nullable; @@ -90,13 +90,7 @@ public abstract class AbstractIgniteDataSource extends AbstractTable /** {@inheritDoc} */ @Override - public RelDataType getRowType(RelDataTypeFactory typeFactory) { - return getRowType(typeFactory, null); - } - - /** {@inheritDoc} */ - @Override - public RelDataType getRowType(RelDataTypeFactory typeFactory, ImmutableBitSet requiredColumns) { + public RelDataType getRowType(RelDataTypeFactory typeFactory, @Nullable ImmutableIntList requiredColumns) { return desc.rowType((IgniteTypeFactory) typeFactory, requiredColumns); } diff --git a/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/schema/IgniteDataSource.java b/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/schema/IgniteDataSource.java index 642155788a7..0dbd746d932 100644 --- a/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/schema/IgniteDataSource.java +++ b/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/schema/IgniteDataSource.java @@ -23,7 +23,9 @@ import org.apache.calcite.rel.type.RelDataTypeFactory; import org.apache.calcite.schema.TranslatableTable; import org.apache.calcite.schema.Wrapper; import org.apache.calcite.util.ImmutableBitSet; +import org.apache.calcite.util.ImmutableIntList; import org.apache.ignite.internal.sql.engine.trait.IgniteDistribution; +import org.jetbrains.annotations.Nullable; /** * Base interface for data sources such as tables and views. @@ -61,7 +63,8 @@ public interface IgniteDataSource extends TranslatableTable, Wrapper { /** {@inheritDoc} */ @Override default RelDataType getRowType(RelDataTypeFactory typeFactory) { - return getRowType(typeFactory, null); + // TODO remove cast + return getRowType(typeFactory, (ImmutableIntList) null); } /** @@ -70,7 +73,16 @@ public interface IgniteDataSource extends TranslatableTable, Wrapper { * @param typeFactory Factory. * @param requiredColumns Used columns enumeration. */ - RelDataType getRowType(RelDataTypeFactory typeFactory, ImmutableBitSet requiredColumns); + // TODO: should this method be removed? + RelDataType getRowType(RelDataTypeFactory typeFactory, @Nullable ImmutableIntList requiredColumns); + + @Deprecated + default RelDataType getRowType(RelDataTypeFactory typeFactory, @Nullable ImmutableBitSet requiredColumns) { + if (requiredColumns == null) { + return getRowType(typeFactory, (ImmutableIntList) null); + } + return getRowType(typeFactory, ImmutableIntList.copyOf(requiredColumns.asList())); + } /** * Returns distribution of this data source. diff --git a/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/schema/IgniteTableImpl.java b/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/schema/IgniteTableImpl.java index 6812683ddf8..17e70aae638 100644 --- a/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/schema/IgniteTableImpl.java +++ b/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/schema/IgniteTableImpl.java @@ -17,11 +17,12 @@ package org.apache.ignite.internal.sql.engine.schema; +import it.unimi.dsi.fastutil.ints.IntArrayList; +import it.unimi.dsi.fastutil.ints.IntList; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.function.Supplier; -import java.util.stream.StreamSupport; import org.apache.calcite.plan.RelOptCluster; import org.apache.calcite.plan.RelOptTable; import org.apache.calcite.plan.RelTraitSet; @@ -30,7 +31,6 @@ import org.apache.calcite.rel.hint.RelHint; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rel.type.RelDataTypeFactory; import org.apache.calcite.schema.Statistic; -import org.apache.calcite.util.ImmutableBitSet; import org.apache.calcite.util.ImmutableIntList; import org.apache.ignite.internal.sql.engine.rel.logical.IgniteLogicalTableScan; import org.apache.ignite.internal.sql.engine.type.IgniteTypeFactory; @@ -43,8 +43,8 @@ import org.jetbrains.annotations.Nullable; */ public class IgniteTableImpl extends AbstractIgniteDataSource implements IgniteTable { private final ImmutableIntList keyColumns; - private final @Nullable ImmutableBitSet columnsToInsert; - private final @Nullable ImmutableBitSet columnsToUpdate; + private final @Nullable ImmutableIntList columnsToInsert; + private final @Nullable ImmutableIntList columnsToUpdate; private final Map<String, IgniteIndex> indexMap; @@ -72,12 +72,7 @@ public class IgniteTableImpl extends AbstractIgniteDataSource implements IgniteT this.partitions = partitions; this.zoneId = zoneId; this.columnsToInsert = deriveColumnsToInsert(desc); - - int virtualColumnsCount = (int) StreamSupport.stream(desc.spliterator(), false) - .filter(ColumnDescriptor::virtual) - .count(); - - this.columnsToUpdate = ImmutableBitSet.range(desc.columnsCount() - virtualColumnsCount); + this.columnsToUpdate = deriveColumnsToUpdate(desc); colocationColumnTypes = new Lazy<>(this::evaluateTypes); } @@ -97,7 +92,7 @@ public class IgniteTableImpl extends AbstractIgniteDataSource implements IgniteT return builder.build(); } - private static @Nullable ImmutableBitSet deriveColumnsToInsert(TableDescriptor desc) { + private static @Nullable ImmutableIntList deriveColumnsToInsert(TableDescriptor desc) { /* Columns to insert are columns which will be expanded in case user omit columns list in insert statement as in example below: @@ -113,7 +108,7 @@ public class IgniteTableImpl extends AbstractIgniteDataSource implements IgniteT See org.apache.ignite.internal.sql.engine.util.Commons.implicitPkEnabled, and org.apache.ignite.internal.sql.engine.schema.SqlSchemaManagerImpl.injectDefault for details. */ - ImmutableBitSet.Builder builder = ImmutableBitSet.builder(); + IntList columnsToInsert = new IntArrayList(desc.columnsCount()); boolean hiddenColumnFound = false; for (ColumnDescriptor columnDescriptor : desc) { @@ -123,11 +118,32 @@ public class IgniteTableImpl extends AbstractIgniteDataSource implements IgniteT continue; } - builder.set(columnDescriptor.logicalIndex()); + columnsToInsert.add(columnDescriptor.logicalIndex()); } if (hiddenColumnFound) { - return builder.build(); + return ImmutableIntList.of(columnsToInsert.toIntArray()); + } + + return null; + } + + private static @Nullable ImmutableIntList deriveColumnsToUpdate(TableDescriptor desc) { + IntList columnsToUpdate = new IntArrayList(desc.columnsCount()); + + boolean virtualColumnFound = false; + for (ColumnDescriptor columnDescriptor : desc) { + if (columnDescriptor.virtual()) { + virtualColumnFound = true; + + continue; + } + + columnsToUpdate.add(columnDescriptor.logicalIndex()); + } + + if (virtualColumnFound) { + return ImmutableIntList.of(columnsToUpdate.toIntArray()); } return null; diff --git a/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/schema/TableDescriptor.java b/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/schema/TableDescriptor.java index c364a6eaa0f..47f408060e8 100644 --- a/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/schema/TableDescriptor.java +++ b/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/schema/TableDescriptor.java @@ -19,7 +19,7 @@ package org.apache.ignite.internal.sql.engine.schema; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.sql2rel.InitializerExpressionFactory; -import org.apache.calcite.util.ImmutableBitSet; +import org.apache.calcite.util.ImmutableIntList; import org.apache.ignite.internal.sql.engine.trait.IgniteDistribution; import org.apache.ignite.internal.sql.engine.type.IgniteTypeFactory; import org.jetbrains.annotations.Nullable; @@ -39,7 +39,7 @@ public interface TableDescriptor extends InitializerExpressionFactory, Iterable< * @param usedColumns Participating columns numeration. * @return Row type. */ - RelDataType rowType(IgniteTypeFactory factory, @Nullable ImmutableBitSet usedColumns); + RelDataType rowType(IgniteTypeFactory factory, @Nullable ImmutableIntList usedColumns); /** Returns type of the row excluding all {@link ColumnDescriptor#hidden() hidden} columns. */ RelDataType rowTypeSansHidden(); diff --git a/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/schema/TableDescriptorImpl.java b/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/schema/TableDescriptorImpl.java index afa3c83b48f..3ae5b01b3c2 100644 --- a/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/schema/TableDescriptorImpl.java +++ b/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/schema/TableDescriptorImpl.java @@ -33,7 +33,7 @@ import org.apache.calcite.rex.RexNode; import org.apache.calcite.schema.ColumnStrategy; import org.apache.calcite.sql2rel.InitializerContext; import org.apache.calcite.sql2rel.NullInitializerExpressionFactory; -import org.apache.calcite.util.ImmutableBitSet; +import org.apache.calcite.util.ImmutableIntList; import org.apache.ignite.internal.sql.engine.sql.fun.IgniteSqlOperatorTable; import org.apache.ignite.internal.sql.engine.trait.IgniteDistribution; import org.apache.ignite.internal.sql.engine.type.IgniteTypeFactory; @@ -152,8 +152,8 @@ public class TableDescriptorImpl extends NullInitializerExpressionFactory implem /** {@inheritDoc} */ @Override - public RelDataType rowType(IgniteTypeFactory factory, @Nullable ImmutableBitSet usedColumns) { - if (usedColumns == null || usedColumns.cardinality() == descriptors.length) { + public RelDataType rowType(IgniteTypeFactory factory, @Nullable ImmutableIntList usedColumns) { + if (usedColumns == null) { return rowType; } else { Builder builder = new Builder(factory); diff --git a/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/util/RowTypeUtils.java b/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/util/RowTypeUtils.java index b0bc589de58..18aaf94849f 100644 --- a/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/util/RowTypeUtils.java +++ b/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/util/RowTypeUtils.java @@ -17,9 +17,9 @@ package org.apache.ignite.internal.sql.engine.util; -import java.util.BitSet; +import it.unimi.dsi.fastutil.ints.IntArrayList; import org.apache.calcite.rel.type.RelDataType; -import org.apache.calcite.util.ImmutableBitSet; +import org.apache.calcite.util.ImmutableIntList; import org.apache.ignite.internal.sql.engine.schema.ColumnDescriptor; import org.apache.ignite.internal.sql.engine.schema.TableDescriptor; import org.apache.ignite.internal.sql.engine.type.IgniteTypeFactory; @@ -46,24 +46,30 @@ public final class RowTypeUtils { * @return Stored rows count. */ public static int storedRowsCount(TableDescriptor tableDescriptor) { - return storedColumns(tableDescriptor).cardinality(); + int count = 0; + for (ColumnDescriptor descriptor : tableDescriptor) { + count += descriptor.virtual() ? 0 : 1; + } + return count; } - private static ImmutableBitSet storedColumns(TableDescriptor tableDescriptor) { - BitSet virtualColumns = new BitSet(); + private static ImmutableIntList storedColumns(TableDescriptor tableDescriptor) { + IntArrayList storedColumns = new IntArrayList(tableDescriptor.columnsCount()); + + boolean virtualColumnFound = false; for (ColumnDescriptor descriptor : tableDescriptor) { - if (descriptor.virtual()) { - virtualColumns.set(descriptor.logicalIndex()); + if (!descriptor.virtual()) { + storedColumns.add(descriptor.logicalIndex()); + } else { + virtualColumnFound = true; } } - ImmutableBitSet storedColumns; - if (virtualColumns.isEmpty()) { - storedColumns = ImmutableBitSet.range(tableDescriptor.columnsCount()); - } else { - virtualColumns.flip(0, tableDescriptor.columnsCount()); - storedColumns = ImmutableBitSet.fromBitSet(virtualColumns); + + + if (virtualColumnFound) { + return ImmutableIntList.of(storedColumns.toIntArray()); } - return storedColumns; + return null; } } diff --git a/modules/sql-engine/src/test/java/org/apache/ignite/internal/sql/engine/exec/ExecutableTableRegistrySelfTest.java b/modules/sql-engine/src/test/java/org/apache/ignite/internal/sql/engine/exec/ExecutableTableRegistrySelfTest.java index ae465b40f9f..4f3600f3043 100644 --- a/modules/sql-engine/src/test/java/org/apache/ignite/internal/sql/engine/exec/ExecutableTableRegistrySelfTest.java +++ b/modules/sql-engine/src/test/java/org/apache/ignite/internal/sql/engine/exec/ExecutableTableRegistrySelfTest.java @@ -157,7 +157,6 @@ public class ExecutableTableRegistrySelfTest extends BaseIgniteAbstractTest { when(schemaManager.schemaRegistry(tableId)).thenReturn(schemaRegistry); when(schemaRegistry.schema(tableVersion)).thenReturn(schemaDescriptor); when(descriptor.iterator()).thenReturn(Collections.emptyIterator()); - when(descriptor.spliterator()).thenReturn(Spliterators.emptySpliterator()); IgniteTable sqlTable = new IgniteTableImpl( "TBL1", tableId, tableVersion, descriptor, ImmutableIntList.of(0), new TestStatistic(1_000.0), Map.of(), 1, 10000 diff --git a/modules/sql-engine/src/test/java/org/apache/ignite/internal/sql/engine/exec/NoOpExecutableTableRegistry.java b/modules/sql-engine/src/test/java/org/apache/ignite/internal/sql/engine/exec/NoOpExecutableTableRegistry.java index 3b700add39f..54fb01d188a 100644 --- a/modules/sql-engine/src/test/java/org/apache/ignite/internal/sql/engine/exec/NoOpExecutableTableRegistry.java +++ b/modules/sql-engine/src/test/java/org/apache/ignite/internal/sql/engine/exec/NoOpExecutableTableRegistry.java @@ -22,6 +22,7 @@ import java.util.List; import java.util.concurrent.CompletableFuture; import java.util.concurrent.Flow.Publisher; import java.util.function.Supplier; +import org.apache.calcite.util.ImmutableIntList; import org.apache.ignite.internal.sql.engine.exec.RowHandler.RowFactory; import org.apache.ignite.internal.sql.engine.exec.exp.RangeCondition; import org.apache.ignite.internal.sql.engine.exec.mapping.ColocationGroup; @@ -53,21 +54,21 @@ public final class NoOpExecutableTableRegistry implements ExecutableTableRegistr return new ScannableTable() { @Override public <RowT> Publisher<RowT> scan(ExecutionContext<RowT> ctx, PartitionWithConsistencyToken partWithConsistencyToken, - RowFactory<RowT> rowFactory, @Nullable BitSet requiredColumns) { + RowFactory<RowT> rowFactory, @Nullable ImmutableIntList requiredColumns) { return SubscriptionUtils.fromIterable(new CompletableFuture<>()); } @Override public <RowT> Publisher<RowT> indexRangeScan(ExecutionContext<RowT> ctx, PartitionWithConsistencyToken partWithConsistencyToken, RowFactory<RowT> rowFactory, int indexId, - List<String> columns, @Nullable RangeCondition<RowT> cond, @Nullable BitSet requiredColumns) { + List<String> columns, @Nullable RangeCondition<RowT> cond, @Nullable ImmutableIntList requiredColumns) { return SubscriptionUtils.fromIterable(new CompletableFuture<>()); } @Override public <RowT> Publisher<RowT> indexLookup(ExecutionContext<RowT> ctx, PartitionWithConsistencyToken partWithConsistencyToken, RowFactory<RowT> rowFactory, int indexId, - List<String> columns, RowT key, @Nullable BitSet requiredColumns) { + List<String> columns, RowT key, @Nullable ImmutableIntList requiredColumns) { return SubscriptionUtils.fromIterable(new CompletableFuture<>()); } diff --git a/modules/sql-engine/src/test/java/org/apache/ignite/internal/sql/engine/exec/ProjectedTableRowConverterSelfTest.java b/modules/sql-engine/src/test/java/org/apache/ignite/internal/sql/engine/exec/ProjectedTableRowConverterSelfTest.java index 09e32e42dc0..20c7f254a56 100644 --- a/modules/sql-engine/src/test/java/org/apache/ignite/internal/sql/engine/exec/ProjectedTableRowConverterSelfTest.java +++ b/modules/sql-engine/src/test/java/org/apache/ignite/internal/sql/engine/exec/ProjectedTableRowConverterSelfTest.java @@ -90,7 +90,7 @@ public class ProjectedTableRowConverterSelfTest extends BaseIgniteAbstractTest { ProjectedTableRowConverterImpl converter = new ProjectedTableRowConverterImpl( schemaRegistry, schema, - BitSets.of(1, 3), + new int[]{1, 3}, Int2ObjectMaps.emptyMap() ); diff --git a/modules/sql-engine/src/test/java/org/apache/ignite/internal/sql/engine/exec/QueryTimeoutTest.java b/modules/sql-engine/src/test/java/org/apache/ignite/internal/sql/engine/exec/QueryTimeoutTest.java index 8cf27f2b70c..f853c4c812f 100644 --- a/modules/sql-engine/src/test/java/org/apache/ignite/internal/sql/engine/exec/QueryTimeoutTest.java +++ b/modules/sql-engine/src/test/java/org/apache/ignite/internal/sql/engine/exec/QueryTimeoutTest.java @@ -30,6 +30,7 @@ import java.util.concurrent.Flow.Publisher; import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Collectors; import java.util.stream.IntStream; +import org.apache.calcite.util.ImmutableIntList; import org.apache.ignite.internal.catalog.CatalogManager; import org.apache.ignite.internal.sql.engine.AsyncSqlCursor; import org.apache.ignite.internal.sql.engine.QueryCancelledException; @@ -217,20 +218,20 @@ public class QueryTimeoutTest extends BaseIgniteAbstractTest { return new ScannableTable() { @Override public <RowT> Publisher<RowT> scan(ExecutionContext<RowT> ctx, PartitionWithConsistencyToken partWithConsistencyToken, - RowFactory<RowT> rowFactory, @Nullable BitSet requiredColumns) { + RowFactory<RowT> rowFactory, @Nullable ImmutableIntList requiredColumns) { return SubscriptionUtils.fromIterable(new CompletableFuture<>()); } @Override public <RowT> Publisher<RowT> indexRangeScan(ExecutionContext<RowT> ctx, PartitionWithConsistencyToken partWithConsistencyToken, RowFactory<RowT> rowFactory, int indexId, List<String> columns, @Nullable RangeCondition<RowT> cond, - @Nullable BitSet requiredColumns) { + @Nullable ImmutableIntList requiredColumns) { return SubscriptionUtils.fromIterable(new CompletableFuture<>()); } @Override public <RowT> Publisher<RowT> indexLookup(ExecutionContext<RowT> ctx, PartitionWithConsistencyToken partWithConsistencyToken, - RowFactory<RowT> rowFactory, int indexId, List<String> columns, RowT key, @Nullable BitSet requiredColumns) { + RowFactory<RowT> rowFactory, int indexId, List<String> columns, RowT key, @Nullable ImmutableIntList requiredColumns) { return SubscriptionUtils.fromIterable(new CompletableFuture<>()); } diff --git a/modules/sql-engine/src/test/java/org/apache/ignite/internal/sql/engine/exec/TableRowConverterSelfTest.java b/modules/sql-engine/src/test/java/org/apache/ignite/internal/sql/engine/exec/TableRowConverterSelfTest.java index 019867afb6d..c3177f4a3ea 100644 --- a/modules/sql-engine/src/test/java/org/apache/ignite/internal/sql/engine/exec/TableRowConverterSelfTest.java +++ b/modules/sql-engine/src/test/java/org/apache/ignite/internal/sql/engine/exec/TableRowConverterSelfTest.java @@ -23,6 +23,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.when; import java.nio.ByteBuffer; +import java.util.BitSet; import java.util.List; import java.util.Map; import java.util.stream.Collectors; @@ -182,7 +183,7 @@ public class TableRowConverterSelfTest extends BaseIgniteAbstractTest { Mockito.mock(TableDescriptor.class), schemaRegistry, schema - ).create(null); + ).create((BitSet) null); BinaryRowEx convertedRow = converter.toKeyRow(executionContext, wrapper); diff --git a/modules/sql-engine/src/test/java/org/apache/ignite/internal/sql/engine/exec/rel/IndexScanNodeExecutionTest.java b/modules/sql-engine/src/test/java/org/apache/ignite/internal/sql/engine/exec/rel/IndexScanNodeExecutionTest.java index a0bbf28e7ca..cee0011618f 100644 --- a/modules/sql-engine/src/test/java/org/apache/ignite/internal/sql/engine/exec/rel/IndexScanNodeExecutionTest.java +++ b/modules/sql-engine/src/test/java/org/apache/ignite/internal/sql/engine/exec/rel/IndexScanNodeExecutionTest.java @@ -43,6 +43,7 @@ import org.apache.calcite.rel.RelFieldCollation; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rel.type.RelDataTypeFactory.Builder; import org.apache.calcite.sql.type.SqlTypeName; +import org.apache.calcite.util.ImmutableIntList; import org.apache.ignite.internal.sql.engine.exec.ExecutionContext; import org.apache.ignite.internal.sql.engine.exec.PartitionProvider; import org.apache.ignite.internal.sql.engine.exec.PartitionWithConsistencyToken; @@ -310,7 +311,7 @@ public class IndexScanNodeExecutionTest extends AbstractExecutionTest<Object[]> ExecutionContext<RowT> ctx, PartitionWithConsistencyToken partWithConsistencyToken, RowFactory<RowT> rowFactory, - @Nullable BitSet requiredColumns + @Nullable ImmutableIntList requiredColumns ) { throw new UnsupportedOperationException("Not supported"); @@ -320,7 +321,7 @@ public class IndexScanNodeExecutionTest extends AbstractExecutionTest<Object[]> @Override public <RowT> Publisher<RowT> indexRangeScan(ExecutionContext<RowT> ctx, PartitionWithConsistencyToken partWithConsistencyToken, RowFactory<RowT> rowFactory, int indexId, List<String> columns, - @Nullable RangeCondition<RowT> cond, @Nullable BitSet requiredColumns) { + @Nullable RangeCondition<RowT> cond, @Nullable ImmutableIntList requiredColumns) { List<T> list = partitionedData.get(partWithConsistencyToken.partId()); return new ScanPublisher<>(list, ctx, rowFactory); @@ -329,7 +330,7 @@ public class IndexScanNodeExecutionTest extends AbstractExecutionTest<Object[]> @Override public <RowT> Publisher<RowT> indexLookup(ExecutionContext<RowT> ctx, PartitionWithConsistencyToken partWithConsistencyToken, RowFactory<RowT> rowFactory, int indexId, List<String> columns, - RowT key, @Nullable BitSet requiredColumns) { + RowT key, @Nullable ImmutableIntList requiredColumns) { return newPublisher(ctx, partWithConsistencyToken, rowFactory); } diff --git a/modules/sql-engine/src/test/java/org/apache/ignite/internal/sql/engine/exec/rel/ScannableTableSelfTest.java b/modules/sql-engine/src/test/java/org/apache/ignite/internal/sql/engine/exec/rel/ScannableTableSelfTest.java index e720d7da9ad..6c0eade045f 100644 --- a/modules/sql-engine/src/test/java/org/apache/ignite/internal/sql/engine/exec/rel/ScannableTableSelfTest.java +++ b/modules/sql-engine/src/test/java/org/apache/ignite/internal/sql/engine/exec/rel/ScannableTableSelfTest.java @@ -53,6 +53,7 @@ import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rel.type.RelDataTypeFactory.Builder; import org.apache.calcite.rel.type.RelDataTypeField; import org.apache.calcite.sql.type.SqlTypeName; +import org.apache.calcite.util.ImmutableIntList; import org.apache.ignite.internal.hlc.HybridTimestamp; import org.apache.ignite.internal.replicator.TablePartitionId; import org.apache.ignite.internal.schema.BinaryRow; @@ -263,8 +264,7 @@ public class ScannableTableSelfTest extends BaseIgniteAbstractTest { input.addRow(binaryRow); Tester tester = new Tester(input); - tester.requiredFields = new BitSet(); - tester.requiredFields.set(1); + tester.requiredFields = ImmutableIntList.of(1); int partitionId = 1; long consistencyToken = 2; @@ -289,7 +289,7 @@ public class ScannableTableSelfTest extends BaseIgniteAbstractTest { nullable(BinaryTuplePrefix.class), nullable(BinaryTuplePrefix.class), anyInt(), - eq(tester.requiredFields), + isNull(), eq(tx.coordinatorId()) ); } else { @@ -305,7 +305,7 @@ public class ScannableTableSelfTest extends BaseIgniteAbstractTest { nullable(BinaryTuplePrefix.class), nullable(BinaryTuplePrefix.class), anyInt(), - eq(tester.requiredFields) + isNull() ); } @@ -326,8 +326,7 @@ public class ScannableTableSelfTest extends BaseIgniteAbstractTest { input.addRow(binaryRow); Tester tester = new Tester(input); - tester.requiredFields = new BitSet(); - tester.requiredFields.set(1); + tester.requiredFields = ImmutableIntList.of(1); int partitionId = 1; long consistencyToken = 2; @@ -411,7 +410,7 @@ public class ScannableTableSelfTest extends BaseIgniteAbstractTest { prefix.capture(), nullable(BinaryTuplePrefix.class), anyInt(), - eq(tester.requiredFields), + isNull(), eq(tx.coordinatorId()) ); } else { @@ -427,7 +426,7 @@ public class ScannableTableSelfTest extends BaseIgniteAbstractTest { prefix.capture(), nullable(BinaryTuplePrefix.class), anyInt(), - eq(tester.requiredFields) + isNull() ); } @@ -501,8 +500,7 @@ public class ScannableTableSelfTest extends BaseIgniteAbstractTest { input.addRow(binaryRow); Tester tester = new Tester(input); - tester.requiredFields = new BitSet(); - tester.requiredFields.set(1); + tester.requiredFields = ImmutableIntList.of(1); int partitionId = 1; long consistencyToken = 2; @@ -586,7 +584,7 @@ public class ScannableTableSelfTest extends BaseIgniteAbstractTest { final RowCollectingTableRowConverter rowConverter; - BitSet requiredFields; + ImmutableIntList requiredFields; Tester(TestInput input) { this.input = input; @@ -696,7 +694,7 @@ public class ScannableTableSelfTest extends BaseIgniteAbstractTest { any(ClusterNode.class), any(Integer.class), nullable(BinaryTuple.class), - nullable(BitSet.class), + isNull(), any(UUID.class)); } else { doAnswer(i -> input.publisher).when(internalTable).lookup( @@ -707,7 +705,7 @@ public class ScannableTableSelfTest extends BaseIgniteAbstractTest { any(PrimaryReplica.class), any(Integer.class), nullable(BinaryTuple.class), - nullable(BitSet.class)); + isNull()); } RowHandler<Object[]> rowHandler = ArrayRowHandler.INSTANCE; diff --git a/modules/sql-engine/src/test/java/org/apache/ignite/internal/sql/engine/framework/TestBuilders.java b/modules/sql-engine/src/test/java/org/apache/ignite/internal/sql/engine/framework/TestBuilders.java index 69b8d65de44..ebb04234f72 100644 --- a/modules/sql-engine/src/test/java/org/apache/ignite/internal/sql/engine/framework/TestBuilders.java +++ b/modules/sql-engine/src/test/java/org/apache/ignite/internal/sql/engine/framework/TestBuilders.java @@ -201,7 +201,7 @@ public class TestBuilders { ExecutionContext<RowT> ctx, PartitionWithConsistencyToken partWithConsistencyToken, RowFactory<RowT> rowFactory, - @Nullable BitSet requiredColumns + @Nullable ImmutableIntList requiredColumns ) { return new TransformingPublisher<>( @@ -228,7 +228,7 @@ public class TestBuilders { ExecutionContext<RowT> ctx, PartitionWithConsistencyToken partWithConsistencyToken, RowFactory<RowT> rowFactory, - @Nullable BitSet requiredColumns + @Nullable ImmutableIntList requiredColumns ) { return new TransformingPublisher<>( @@ -255,7 +255,7 @@ public class TestBuilders { ExecutionContext<RowT> ctx, PartitionWithConsistencyToken partWithConsistencyToken, RowFactory<RowT> rowFactory, - @Nullable BitSet requiredColumns + @Nullable ImmutableIntList requiredColumns ) { throw new UnsupportedOperationException(); } @@ -263,7 +263,7 @@ public class TestBuilders { @Override public <RowT> Publisher<RowT> indexRangeScan(ExecutionContext<RowT> ctx, PartitionWithConsistencyToken partWithConsistencyToken, RowFactory<RowT> rowFactory, int indexId, List<String> columns, @Nullable RangeCondition<RowT> cond, - @Nullable BitSet requiredColumns) { + @Nullable ImmutableIntList requiredColumns) { return new TransformingPublisher<>( SubscriptionUtils.fromIterable( () -> new TransformingIterator<>( @@ -277,7 +277,7 @@ public class TestBuilders { @Override public <RowT> Publisher<RowT> indexLookup(ExecutionContext<RowT> ctx, PartitionWithConsistencyToken partWithConsistencyToken, - RowFactory<RowT> rowFactory, int indexId, List<String> columns, RowT key, @Nullable BitSet requiredColumns) { + RowFactory<RowT> rowFactory, int indexId, List<String> columns, RowT key, @Nullable ImmutableIntList requiredColumns) { throw new UnsupportedOperationException(); } @@ -305,7 +305,7 @@ public class TestBuilders { ExecutionContext<RowT> ctx, PartitionWithConsistencyToken partWithConsistencyToken, RowFactory<RowT> rowFactory, - @Nullable BitSet requiredColumns + @Nullable ImmutableIntList requiredColumns ) { throw new UnsupportedOperationException(); } @@ -313,13 +313,13 @@ public class TestBuilders { @Override public <RowT> Publisher<RowT> indexRangeScan(ExecutionContext<RowT> ctx, PartitionWithConsistencyToken partWithConsistencyToken, RowFactory<RowT> rowFactory, int indexId, List<String> columns, @Nullable RangeCondition<RowT> cond, - @Nullable BitSet requiredColumns) { + @Nullable ImmutableIntList requiredColumns) { throw new UnsupportedOperationException(); } @Override public <RowT> Publisher<RowT> indexLookup(ExecutionContext<RowT> ctx, PartitionWithConsistencyToken partWithConsistencyToken, - RowFactory<RowT> rowFactory, int indexId, List<String> columns, RowT key, @Nullable BitSet requiredColumns) { + RowFactory<RowT> rowFactory, int indexId, List<String> columns, RowT key, @Nullable ImmutableIntList requiredColumns) { return new TransformingPublisher<>( SubscriptionUtils.fromIterable( () -> new TransformingIterator<>( @@ -1712,16 +1712,22 @@ public class TestBuilders { return builder.build(); } + @Deprecated private static Object[] project(Object[] row, @Nullable BitSet requiredElements) { if (requiredElements == null) { return row; } + return project(row, ImmutableIntList.copyOf(requiredElements.stream().iterator())); + } + private static Object[] project(Object[] row, @Nullable ImmutableIntList requiredElements) { + if (requiredElements == null) { + return row; + } - Object[] newRow = new Object[requiredElements.cardinality()]; + Object[] newRow = new Object[requiredElements.size()]; - int idx = 0; - for (int i = requiredElements.nextSetBit(0); i != -1; i = requiredElements.nextSetBit(i + 1)) { - newRow[idx++] = row[i]; + for (int i = 0; i < requiredElements.size(); i++) { + newRow[i] = row[requiredElements.get(i)]; } return newRow; @@ -1882,7 +1888,7 @@ public class TestBuilders { ExecutionContext<RowT> ctx, PartitionWithConsistencyToken partWithConsistencyToken, RowFactory<RowT> rowFactory, - @Nullable BitSet requiredColumns + @Nullable ImmutableIntList requiredColumns ) { throw new UnsupportedOperationException(); } @@ -1890,13 +1896,13 @@ public class TestBuilders { @Override public <RowT> Publisher<RowT> indexRangeScan(ExecutionContext<RowT> ctx, PartitionWithConsistencyToken partWithConsistencyToken, RowFactory<RowT> rowFactory, int indexId, List<String> columns, @Nullable RangeCondition<RowT> cond, - @Nullable BitSet requiredColumns) { + @Nullable ImmutableIntList requiredColumns) { throw new UnsupportedOperationException(); } @Override public <RowT> Publisher<RowT> indexLookup(ExecutionContext<RowT> ctx, PartitionWithConsistencyToken partWithConsistencyToken, - RowFactory<RowT> rowFactory, int indexId, List<String> columns, RowT key, @Nullable BitSet requiredColumns) { + RowFactory<RowT> rowFactory, int indexId, List<String> columns, RowT key, @Nullable ImmutableIntList requiredColumns) { throw new UnsupportedOperationException(); } diff --git a/modules/sql-engine/src/test/java/org/apache/ignite/internal/sql/engine/framework/TestClusterTest.java b/modules/sql-engine/src/test/java/org/apache/ignite/internal/sql/engine/framework/TestClusterTest.java index 1da45bdcd20..64ebe6b8469 100644 --- a/modules/sql-engine/src/test/java/org/apache/ignite/internal/sql/engine/framework/TestClusterTest.java +++ b/modules/sql-engine/src/test/java/org/apache/ignite/internal/sql/engine/framework/TestClusterTest.java @@ -39,6 +39,7 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.Flow.Publisher; import java.util.stream.Collectors; import java.util.stream.IntStream; +import org.apache.calcite.util.ImmutableIntList; import org.apache.ignite.internal.hlc.HybridClock; import org.apache.ignite.internal.sql.engine.InternalSqlRow; import org.apache.ignite.internal.sql.engine.exec.AsyncDataCursor; @@ -79,7 +80,7 @@ public class TestClusterTest extends BaseIgniteAbstractTest { ExecutionContext<RowT> ctx, PartitionWithConsistencyToken partWithConsistencyToken, RowFactory<RowT> rowFactory, - @Nullable BitSet requiredColumns + @Nullable ImmutableIntList requiredColumns ) { return new TransformingPublisher<>( @@ -94,7 +95,7 @@ public class TestClusterTest extends BaseIgniteAbstractTest { @Override public <RowT> Publisher<RowT> indexRangeScan(ExecutionContext<RowT> ctx, PartitionWithConsistencyToken partWithConsistencyToken, RowFactory<RowT> rowFactory, int indexId, List<String> columns, @Nullable RangeCondition<RowT> cond, - @Nullable BitSet requiredColumns) { + @Nullable ImmutableIntList requiredColumns) { return new TransformingPublisher<>( SubscriptionUtils.fromIterable( @@ -107,7 +108,7 @@ public class TestClusterTest extends BaseIgniteAbstractTest { @Override public <RowT> Publisher<RowT> indexLookup(ExecutionContext<RowT> ctx, PartitionWithConsistencyToken partWithConsistencyToken, - RowFactory<RowT> rowFactory, int indexId, List<String> columns, RowT key, @Nullable BitSet requiredColumns) { + RowFactory<RowT> rowFactory, int indexId, List<String> columns, RowT key, @Nullable ImmutableIntList requiredColumns) { return new TransformingPublisher<>( SubscriptionUtils.fromIterable( diff --git a/modules/sql-engine/src/test/java/org/apache/ignite/internal/sql/engine/planner/AbstractPlannerTest.java b/modules/sql-engine/src/test/java/org/apache/ignite/internal/sql/engine/planner/AbstractPlannerTest.java index 242a67a3dd7..f62124ade13 100644 --- a/modules/sql-engine/src/test/java/org/apache/ignite/internal/sql/engine/planner/AbstractPlannerTest.java +++ b/modules/sql-engine/src/test/java/org/apache/ignite/internal/sql/engine/planner/AbstractPlannerTest.java @@ -78,6 +78,7 @@ import org.apache.calcite.sql2rel.InitializerContext; import org.apache.calcite.sql2rel.SqlToRelConverter; import org.apache.calcite.tools.Frameworks; import org.apache.calcite.util.ImmutableBitSet; +import org.apache.calcite.util.ImmutableIntList; import org.apache.calcite.util.TimeString; import org.apache.calcite.util.TimestampString; import org.apache.calcite.util.Util; @@ -855,7 +856,7 @@ public abstract class AbstractPlannerTest extends IgniteAbstractTest { /** {@inheritDoc} */ @Override - public RelDataType rowType(IgniteTypeFactory factory, ImmutableBitSet usedColumns) { + public RelDataType rowType(IgniteTypeFactory factory, ImmutableIntList usedColumns) { return rowType; } diff --git a/modules/sql-engine/src/test/java/org/apache/ignite/internal/sql/engine/planner/PlannerTimeoutTest.java b/modules/sql-engine/src/test/java/org/apache/ignite/internal/sql/engine/planner/PlannerTimeoutTest.java index 7b1713b8a86..c7cdc4b988f 100644 --- a/modules/sql-engine/src/test/java/org/apache/ignite/internal/sql/engine/planner/PlannerTimeoutTest.java +++ b/modules/sql-engine/src/test/java/org/apache/ignite/internal/sql/engine/planner/PlannerTimeoutTest.java @@ -35,6 +35,8 @@ import org.apache.calcite.plan.volcano.VolcanoPlanner; import org.apache.calcite.plan.volcano.VolcanoTimeoutException; import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.RelVisitor; +import org.apache.calcite.util.ImmutableBitSet; +import org.apache.calcite.util.ImmutableIntList; import org.apache.ignite.internal.metrics.MetricManagerImpl; import org.apache.ignite.internal.sql.engine.SqlOperationContext; import org.apache.ignite.internal.sql.engine.framework.PredefinedSchemaManager; @@ -160,8 +162,8 @@ public class PlannerTimeoutTest extends AbstractPlannerTest { throw new RuntimeException(e); } // Call original method. - return igniteTable.getRowType(inv.getArgument(0), inv.getArgument(1)); - }).when(spyTable).getRowType(any(), any()); + return igniteTable.getRowType(inv.getArgument(0), (ImmutableIntList) inv.getArgument(1)); + }).when(spyTable).getRowType(any(), any(ImmutableIntList.class)); return spyTable; } diff --git a/modules/sql-engine/src/test/java/org/apache/ignite/internal/sql/engine/prepare/PrepareServiceImplTest.java b/modules/sql-engine/src/test/java/org/apache/ignite/internal/sql/engine/prepare/PrepareServiceImplTest.java index f2b8a09a3f5..717f7108a59 100644 --- a/modules/sql-engine/src/test/java/org/apache/ignite/internal/sql/engine/prepare/PrepareServiceImplTest.java +++ b/modules/sql-engine/src/test/java/org/apache/ignite/internal/sql/engine/prepare/PrepareServiceImplTest.java @@ -42,6 +42,7 @@ import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.stream.Collectors; import java.util.stream.Stream; +import org.apache.calcite.util.ImmutableIntList; import org.apache.ignite.internal.hlc.HybridClockImpl; import org.apache.ignite.internal.metrics.MetricManagerImpl; import org.apache.ignite.internal.sql.SqlCommon; @@ -302,8 +303,8 @@ public class PrepareServiceImplTest extends BaseIgniteAbstractTest { throw new RuntimeException(e); } // Call original method. - return igniteTable.getRowType(inv.getArgument(0), inv.getArgument(1)); - }).when(spyTable).getRowType(any(), any()); + return igniteTable.getRowType(inv.getArgument(0), (ImmutableIntList) inv.getArgument(1)); + }).when(spyTable).getRowType(any(), any(ImmutableIntList.class)); IgniteSchema schema = new IgniteSchema("PUBLIC", 0, List.of(igniteTable)); Cache<Object, Object> cache = CaffeineCacheFactory.INSTANCE.create(100);