xtern commented on code in PR #2568: URL: https://github.com/apache/ignite-3/pull/2568#discussion_r1324210568
########## modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/util/ProjectedTuple.java: ########## @@ -0,0 +1,322 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.sql.engine.util; + +import java.math.BigDecimal; +import java.math.BigInteger; +import java.nio.ByteBuffer; +import java.time.Instant; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.LocalTime; +import java.util.BitSet; +import java.util.Objects; +import java.util.UUID; +import org.apache.ignite.internal.binarytuple.BinaryTupleBuilder; +import org.apache.ignite.internal.binarytuple.BinaryTupleParser; +import org.apache.ignite.internal.binarytuple.BinaryTupleParser.Sink; +import org.apache.ignite.internal.schema.BinaryRowConverter; +import org.apache.ignite.internal.schema.BinaryTuple; +import org.apache.ignite.internal.schema.BinaryTupleSchema; +import org.apache.ignite.internal.schema.BinaryTupleSchema.Element; +import org.apache.ignite.internal.schema.row.InternalTuple; +import org.jetbrains.annotations.Nullable; + +/** + * A facade that creates projection of the given tuple. + * + * <p>Not thread safe! + * + * <p>This projection is used to change indexes of column in original tuple, or to trim + * few columns from original tuple. Here are a few examples:<pre> + * Having tuple ['foo', 'bar', 'baz'], we can + * + * - reorder fields with mapping [2, 1, 0] to get equivalent tuple ['baz', 'bar', 'foo'] + * - or trim certain fields with mapping [0, 2] to get equivalent tuple ['foo', 'baz'] + * - or even repeat some fields with mapping [0, 0, 0] to get equivalent tuple ['foo', 'foo', 'foo'] + * </pre> + */ +public class ProjectedTuple implements InternalTuple { + private final @Nullable BinaryTupleSchema schema; + + private InternalTuple delegate; + private int[] projection; + + private boolean normalized = false; + + /** + * Creates projected tuple with not optimal but reliable conversion. + * + * <p>When call to {@link #byteBuffer()}, the original tuple will be read field by field with regard to provided projection. + * Such an approach had an additional overhead on (de-)serialization fields value, but had no requirement for the tuple + * to be compatible with Binary Tuple format. + * + * @param schema A schema of the original tuple (represented by delegate). Used to read content of the delegate to build a + * proper byte buffer which content satisfying the schema with regard to given projection. + * @param delegate An original tuple to create projection from. + * @param projection A projection. That is, desired order of fields in original tuple. In that projection, index of the array is + * an index of field in resulting projection, and an element of the array at that index is an index of column in original + * tuple. + */ + public ProjectedTuple( + BinaryTupleSchema schema, + InternalTuple delegate, + int[] projection + ) { + this.schema = Objects.requireNonNull(schema); + this.delegate = delegate; + this.projection = projection; + } + + /** + * Creates projected tuple with optimized conversion. + * + * <p>When call to {@link #byteBuffer()}, the original tuple will be rebuild with regard to provided projection + * by copying raw bytes from original tuple. Although this works more optimal, it requires an original tuple + * to be crafted with regard to Binary Tuple format. + * + * <p>It's up to the caller to get sure that provided tuple respect the format. + * + * @param delegate An original tuple to create projection from. + * @param projection A projection. That is, desired order of fields in original tuple. In that projection, index of the array is + * an index of field in resulting projection, and an element of the array at that index is an index of column in original + * tuple. + */ + public ProjectedTuple( + InternalTuple delegate, + int[] projection + ) { + this.delegate = delegate; + this.projection = projection; + + this.schema = null; + } + + @Override + public int elementCount() { + return projection.length; + } + + @Override + public boolean hasNullValue(int col) { + return delegate.hasNullValue(projection[col]); + } + + @Override + public boolean booleanValue(int col) { + return delegate.booleanValue(projection[col]); + } + + @Override + public Boolean booleanValueBoxed(int col) { + return delegate.booleanValueBoxed(projection[col]); + } + + @Override + public byte byteValue(int col) { + return delegate.byteValue(projection[col]); + } + + @Override + public Byte byteValueBoxed(int col) { + return delegate.byteValueBoxed(projection[col]); + } + + @Override + public short shortValue(int col) { + return delegate.shortValue(projection[col]); + } + + @Override + public Short shortValueBoxed(int col) { + return delegate.shortValueBoxed(projection[col]); + } + + @Override + public int intValue(int col) { + return delegate.intValue(projection[col]); + } + + @Override + public Integer intValueBoxed(int col) { + return delegate.intValueBoxed(projection[col]); + } + + @Override + public long longValue(int col) { + return delegate.longValue(projection[col]); + } + + @Override + public Long longValueBoxed(int col) { + return delegate.longValueBoxed(projection[col]); + } + + @Override + public float floatValue(int col) { + return delegate.floatValue(projection[col]); + } + + @Override + public Float floatValueBoxed(int col) { + return delegate.floatValueBoxed(projection[col]); + } + + @Override + public double doubleValue(int col) { + return delegate.doubleValue(projection[col]); + } + + @Override + public Double doubleValueBoxed(int col) { + return delegate.doubleValueBoxed(projection[col]); + } + + @Override + public BigDecimal decimalValue(int col, int decimalScale) { + return delegate.decimalValue(projection[col], decimalScale); + } + + @Override + public BigInteger numberValue(int col) { + return delegate.numberValue(projection[col]); + } + + @Override + public String stringValue(int col) { + return delegate.stringValue(projection[col]); + } + + @Override + public byte[] bytesValue(int col) { + return delegate.bytesValue(projection[col]); + } + + @Override + public UUID uuidValue(int col) { + return delegate.uuidValue(projection[col]); + } + + @Override + public BitSet bitmaskValue(int col) { + return delegate.bitmaskValue(projection[col]); + } + + @Override + public LocalDate dateValue(int col) { + return delegate.dateValue(projection[col]); + } + + @Override + public LocalTime timeValue(int col) { + return delegate.timeValue(projection[col]); + } + + @Override + public LocalDateTime dateTimeValue(int col) { + return delegate.dateTimeValue(projection[col]); + } + + @Override + public Instant timestampValue(int col) { + return delegate.timestampValue(projection[col]); + } + + @Override + public ByteBuffer byteBuffer() { + normalizeIfNeeded(); + + return delegate.byteBuffer(); + } + + private void normalizeIfNeeded() { + if (normalized) { + return; + } + + if (schema != null) { + normalizeSlow(); + } else { + normalizeFast(); + } + } + + private void normalizeSlow() { + assert schema != null; + + var builder = new BinaryTupleBuilder(projection.length); + var newProjection = new int[projection.length]; + + for (int i = 0; i < projection.length; i++) { + int col = projection[i]; + + newProjection[i] = i; + + Element element = schema.element(col); + + BinaryRowConverter.appendValue(builder, element, schema.value(delegate, col)); + } + + delegate = new BinaryTuple(projection.length, builder.build()); + projection = newProjection; + normalized = true; Review Comment: seems `nomalized` flag should be moved to `normalizeIfNeeded` ########## modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/util/ProjectedTuple.java: ########## @@ -0,0 +1,322 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.sql.engine.util; + +import java.math.BigDecimal; +import java.math.BigInteger; +import java.nio.ByteBuffer; +import java.time.Instant; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.LocalTime; +import java.util.BitSet; +import java.util.Objects; +import java.util.UUID; +import org.apache.ignite.internal.binarytuple.BinaryTupleBuilder; +import org.apache.ignite.internal.binarytuple.BinaryTupleParser; +import org.apache.ignite.internal.binarytuple.BinaryTupleParser.Sink; +import org.apache.ignite.internal.schema.BinaryRowConverter; +import org.apache.ignite.internal.schema.BinaryTuple; +import org.apache.ignite.internal.schema.BinaryTupleSchema; +import org.apache.ignite.internal.schema.BinaryTupleSchema.Element; +import org.apache.ignite.internal.schema.row.InternalTuple; +import org.jetbrains.annotations.Nullable; + +/** + * A facade that creates projection of the given tuple. + * + * <p>Not thread safe! + * + * <p>This projection is used to change indexes of column in original tuple, or to trim + * few columns from original tuple. Here are a few examples:<pre> + * Having tuple ['foo', 'bar', 'baz'], we can + * + * - reorder fields with mapping [2, 1, 0] to get equivalent tuple ['baz', 'bar', 'foo'] + * - or trim certain fields with mapping [0, 2] to get equivalent tuple ['foo', 'baz'] + * - or even repeat some fields with mapping [0, 0, 0] to get equivalent tuple ['foo', 'foo', 'foo'] + * </pre> + */ +public class ProjectedTuple implements InternalTuple { + private final @Nullable BinaryTupleSchema schema; + + private InternalTuple delegate; + private int[] projection; + + private boolean normalized = false; + + /** + * Creates projected tuple with not optimal but reliable conversion. + * + * <p>When call to {@link #byteBuffer()}, the original tuple will be read field by field with regard to provided projection. + * Such an approach had an additional overhead on (de-)serialization fields value, but had no requirement for the tuple + * to be compatible with Binary Tuple format. + * + * @param schema A schema of the original tuple (represented by delegate). Used to read content of the delegate to build a + * proper byte buffer which content satisfying the schema with regard to given projection. + * @param delegate An original tuple to create projection from. + * @param projection A projection. That is, desired order of fields in original tuple. In that projection, index of the array is + * an index of field in resulting projection, and an element of the array at that index is an index of column in original + * tuple. + */ + public ProjectedTuple( + BinaryTupleSchema schema, + InternalTuple delegate, + int[] projection + ) { + this.schema = Objects.requireNonNull(schema); + this.delegate = delegate; + this.projection = projection; + } + + /** + * Creates projected tuple with optimized conversion. + * + * <p>When call to {@link #byteBuffer()}, the original tuple will be rebuild with regard to provided projection + * by copying raw bytes from original tuple. Although this works more optimal, it requires an original tuple + * to be crafted with regard to Binary Tuple format. + * + * <p>It's up to the caller to get sure that provided tuple respect the format. + * + * @param delegate An original tuple to create projection from. + * @param projection A projection. That is, desired order of fields in original tuple. In that projection, index of the array is + * an index of field in resulting projection, and an element of the array at that index is an index of column in original + * tuple. + */ + public ProjectedTuple( + InternalTuple delegate, + int[] projection + ) { + this.delegate = delegate; + this.projection = projection; + + this.schema = null; + } + + @Override + public int elementCount() { + return projection.length; + } + + @Override + public boolean hasNullValue(int col) { + return delegate.hasNullValue(projection[col]); + } + + @Override + public boolean booleanValue(int col) { + return delegate.booleanValue(projection[col]); + } + + @Override + public Boolean booleanValueBoxed(int col) { + return delegate.booleanValueBoxed(projection[col]); + } + + @Override + public byte byteValue(int col) { + return delegate.byteValue(projection[col]); + } + + @Override + public Byte byteValueBoxed(int col) { + return delegate.byteValueBoxed(projection[col]); + } + + @Override + public short shortValue(int col) { + return delegate.shortValue(projection[col]); + } + + @Override + public Short shortValueBoxed(int col) { + return delegate.shortValueBoxed(projection[col]); + } + + @Override + public int intValue(int col) { + return delegate.intValue(projection[col]); + } + + @Override + public Integer intValueBoxed(int col) { + return delegate.intValueBoxed(projection[col]); + } + + @Override + public long longValue(int col) { + return delegate.longValue(projection[col]); + } + + @Override + public Long longValueBoxed(int col) { + return delegate.longValueBoxed(projection[col]); + } + + @Override + public float floatValue(int col) { + return delegate.floatValue(projection[col]); + } + + @Override + public Float floatValueBoxed(int col) { + return delegate.floatValueBoxed(projection[col]); + } + + @Override + public double doubleValue(int col) { + return delegate.doubleValue(projection[col]); + } + + @Override + public Double doubleValueBoxed(int col) { + return delegate.doubleValueBoxed(projection[col]); + } + + @Override + public BigDecimal decimalValue(int col, int decimalScale) { + return delegate.decimalValue(projection[col], decimalScale); + } + + @Override + public BigInteger numberValue(int col) { + return delegate.numberValue(projection[col]); + } + + @Override + public String stringValue(int col) { + return delegate.stringValue(projection[col]); + } + + @Override + public byte[] bytesValue(int col) { + return delegate.bytesValue(projection[col]); + } + + @Override + public UUID uuidValue(int col) { + return delegate.uuidValue(projection[col]); + } + + @Override + public BitSet bitmaskValue(int col) { + return delegate.bitmaskValue(projection[col]); + } + + @Override + public LocalDate dateValue(int col) { + return delegate.dateValue(projection[col]); + } + + @Override + public LocalTime timeValue(int col) { + return delegate.timeValue(projection[col]); + } + + @Override + public LocalDateTime dateTimeValue(int col) { + return delegate.dateTimeValue(projection[col]); + } + + @Override + public Instant timestampValue(int col) { + return delegate.timestampValue(projection[col]); + } + + @Override + public ByteBuffer byteBuffer() { + normalizeIfNeeded(); + + return delegate.byteBuffer(); + } + + private void normalizeIfNeeded() { + if (normalized) { + return; + } + + if (schema != null) { + normalizeSlow(); + } else { + normalizeFast(); + } + } + + private void normalizeSlow() { + assert schema != null; + + var builder = new BinaryTupleBuilder(projection.length); + var newProjection = new int[projection.length]; + + for (int i = 0; i < projection.length; i++) { + int col = projection[i]; + + newProjection[i] = i; + + Element element = schema.element(col); + + BinaryRowConverter.appendValue(builder, element, schema.value(delegate, col)); + } + + delegate = new BinaryTuple(projection.length, builder.build()); + projection = newProjection; + normalized = true; + } + + private void normalizeFast() { + var newProjection = new int[projection.length]; + ByteBuffer tupleBuffer = delegate.byteBuffer(); + int[] requiredColumns = projection; + + var parser = new BinaryTupleParser(delegate.elementCount(), tupleBuffer); + + // Estimate total data size. + var stats = new Sink() { + int estimatedValueSize = 0; + + @Override + public void nextElement(int index, int begin, int end) { + estimatedValueSize += end - begin; + } + }; + + for (int columnIndex : requiredColumns) { + parser.fetch(columnIndex, stats); + } + + // Now compose the tuple. + BinaryTupleBuilder builder = new BinaryTupleBuilder(requiredColumns.length, stats.estimatedValueSize); + + int pos = 0; + + for (int columnIndex : requiredColumns) { + parser.fetch(columnIndex, (index, begin, end) -> { + if (begin == end) { + builder.appendNull(); + } else { + builder.appendElementBytes(tupleBuffer, begin, end - begin); + } + }); + + newProjection[pos++] = columnIndex; + } Review Comment: I suggest to remove `pos` counter. ```suggestion for (int i = 0; i < requiredColumns.length; i++) { parser.fetch(requiredColumns[i], (index, begin, end) -> { if (begin == end) { builder.appendNull(); } else { builder.appendElementBytes(tupleBuffer, begin, end - begin); } }); newProjection[i] = requiredColumns[i]; } ``` ########## modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/util/ProjectedTuple.java: ########## @@ -0,0 +1,322 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.sql.engine.util; + +import java.math.BigDecimal; +import java.math.BigInteger; +import java.nio.ByteBuffer; +import java.time.Instant; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.LocalTime; +import java.util.BitSet; +import java.util.Objects; +import java.util.UUID; +import org.apache.ignite.internal.binarytuple.BinaryTupleBuilder; +import org.apache.ignite.internal.binarytuple.BinaryTupleParser; +import org.apache.ignite.internal.binarytuple.BinaryTupleParser.Sink; +import org.apache.ignite.internal.schema.BinaryRowConverter; +import org.apache.ignite.internal.schema.BinaryTuple; +import org.apache.ignite.internal.schema.BinaryTupleSchema; +import org.apache.ignite.internal.schema.BinaryTupleSchema.Element; +import org.apache.ignite.internal.schema.row.InternalTuple; +import org.jetbrains.annotations.Nullable; + +/** + * A facade that creates projection of the given tuple. + * + * <p>Not thread safe! + * + * <p>This projection is used to change indexes of column in original tuple, or to trim + * few columns from original tuple. Here are a few examples:<pre> + * Having tuple ['foo', 'bar', 'baz'], we can + * + * - reorder fields with mapping [2, 1, 0] to get equivalent tuple ['baz', 'bar', 'foo'] + * - or trim certain fields with mapping [0, 2] to get equivalent tuple ['foo', 'baz'] + * - or even repeat some fields with mapping [0, 0, 0] to get equivalent tuple ['foo', 'foo', 'foo'] + * </pre> + */ +public class ProjectedTuple implements InternalTuple { + private final @Nullable BinaryTupleSchema schema; + + private InternalTuple delegate; + private int[] projection; + + private boolean normalized = false; + + /** + * Creates projected tuple with not optimal but reliable conversion. + * + * <p>When call to {@link #byteBuffer()}, the original tuple will be read field by field with regard to provided projection. + * Such an approach had an additional overhead on (de-)serialization fields value, but had no requirement for the tuple + * to be compatible with Binary Tuple format. + * + * @param schema A schema of the original tuple (represented by delegate). Used to read content of the delegate to build a + * proper byte buffer which content satisfying the schema with regard to given projection. + * @param delegate An original tuple to create projection from. + * @param projection A projection. That is, desired order of fields in original tuple. In that projection, index of the array is + * an index of field in resulting projection, and an element of the array at that index is an index of column in original + * tuple. + */ + public ProjectedTuple( + BinaryTupleSchema schema, + InternalTuple delegate, + int[] projection + ) { + this.schema = Objects.requireNonNull(schema); + this.delegate = delegate; + this.projection = projection; + } + + /** + * Creates projected tuple with optimized conversion. + * + * <p>When call to {@link #byteBuffer()}, the original tuple will be rebuild with regard to provided projection + * by copying raw bytes from original tuple. Although this works more optimal, it requires an original tuple + * to be crafted with regard to Binary Tuple format. + * + * <p>It's up to the caller to get sure that provided tuple respect the format. + * + * @param delegate An original tuple to create projection from. + * @param projection A projection. That is, desired order of fields in original tuple. In that projection, index of the array is + * an index of field in resulting projection, and an element of the array at that index is an index of column in original + * tuple. + */ + public ProjectedTuple( + InternalTuple delegate, + int[] projection + ) { + this.delegate = delegate; + this.projection = projection; + + this.schema = null; + } + + @Override + public int elementCount() { + return projection.length; + } + + @Override + public boolean hasNullValue(int col) { + return delegate.hasNullValue(projection[col]); + } + + @Override + public boolean booleanValue(int col) { + return delegate.booleanValue(projection[col]); + } + + @Override + public Boolean booleanValueBoxed(int col) { + return delegate.booleanValueBoxed(projection[col]); + } + + @Override + public byte byteValue(int col) { + return delegate.byteValue(projection[col]); + } + + @Override + public Byte byteValueBoxed(int col) { + return delegate.byteValueBoxed(projection[col]); + } + + @Override + public short shortValue(int col) { + return delegate.shortValue(projection[col]); + } + + @Override + public Short shortValueBoxed(int col) { + return delegate.shortValueBoxed(projection[col]); + } + + @Override + public int intValue(int col) { + return delegate.intValue(projection[col]); + } + + @Override + public Integer intValueBoxed(int col) { + return delegate.intValueBoxed(projection[col]); + } + + @Override + public long longValue(int col) { + return delegate.longValue(projection[col]); + } + + @Override + public Long longValueBoxed(int col) { + return delegate.longValueBoxed(projection[col]); + } + + @Override + public float floatValue(int col) { + return delegate.floatValue(projection[col]); + } + + @Override + public Float floatValueBoxed(int col) { + return delegate.floatValueBoxed(projection[col]); + } + + @Override + public double doubleValue(int col) { + return delegate.doubleValue(projection[col]); + } + + @Override + public Double doubleValueBoxed(int col) { + return delegate.doubleValueBoxed(projection[col]); + } + + @Override + public BigDecimal decimalValue(int col, int decimalScale) { + return delegate.decimalValue(projection[col], decimalScale); + } + + @Override + public BigInteger numberValue(int col) { + return delegate.numberValue(projection[col]); + } + + @Override + public String stringValue(int col) { + return delegate.stringValue(projection[col]); + } + + @Override + public byte[] bytesValue(int col) { + return delegate.bytesValue(projection[col]); + } + + @Override + public UUID uuidValue(int col) { + return delegate.uuidValue(projection[col]); + } + + @Override + public BitSet bitmaskValue(int col) { + return delegate.bitmaskValue(projection[col]); + } + + @Override + public LocalDate dateValue(int col) { + return delegate.dateValue(projection[col]); + } + + @Override + public LocalTime timeValue(int col) { + return delegate.timeValue(projection[col]); + } + + @Override + public LocalDateTime dateTimeValue(int col) { + return delegate.dateTimeValue(projection[col]); + } + + @Override + public Instant timestampValue(int col) { + return delegate.timestampValue(projection[col]); + } + + @Override + public ByteBuffer byteBuffer() { + normalizeIfNeeded(); + + return delegate.byteBuffer(); + } + + private void normalizeIfNeeded() { + if (normalized) { + return; + } + + if (schema != null) { + normalizeSlow(); + } else { + normalizeFast(); + } + } + + private void normalizeSlow() { + assert schema != null; + + var builder = new BinaryTupleBuilder(projection.length); + var newProjection = new int[projection.length]; + + for (int i = 0; i < projection.length; i++) { + int col = projection[i]; + + newProjection[i] = i; + + Element element = schema.element(col); + + BinaryRowConverter.appendValue(builder, element, schema.value(delegate, col)); + } + + delegate = new BinaryTuple(projection.length, builder.build()); + projection = newProjection; + normalized = true; + } + + private void normalizeFast() { + var newProjection = new int[projection.length]; + ByteBuffer tupleBuffer = delegate.byteBuffer(); + int[] requiredColumns = projection; Review Comment: seems this var is redundant p.s. I know this is the code I suggested, but it looks like it needs to be cleaned up :roll_eyes: ########## modules/sql-engine/src/test/java/org/apache/ignite/internal/sql/engine/util/ProjectedTupleTest.java: ########## @@ -0,0 +1,175 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.sql.engine.util; Review Comment: It might be worth moving this test to the `org.apache.ignite.internal.sql.engine.exec.row` package :thinking: ########## modules/sql-engine/src/test/java/org/apache/ignite/internal/sql/engine/util/ProjectedTupleTest.java: ########## @@ -0,0 +1,175 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.sql.engine.util; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.empty; +import static org.hamcrest.Matchers.equalTo; + +import java.util.EnumSet; +import java.util.List; +import java.util.Random; +import java.util.concurrent.ThreadLocalRandom; +import java.util.stream.Collectors; +import java.util.stream.IntStream; +import org.apache.ignite.internal.binarytuple.BinaryTupleBuilder; +import org.apache.ignite.internal.logger.IgniteLogger; +import org.apache.ignite.internal.logger.Loggers; +import org.apache.ignite.internal.schema.BinaryRowConverter; +import org.apache.ignite.internal.schema.BinaryTuple; +import org.apache.ignite.internal.schema.BinaryTupleSchema; +import org.apache.ignite.internal.schema.BinaryTupleSchema.Element; +import org.apache.ignite.internal.schema.NativeType; +import org.apache.ignite.internal.schema.NativeTypeSpec; +import org.apache.ignite.internal.schema.NativeTypes; +import org.apache.ignite.internal.schema.SchemaTestUtils; +import org.apache.ignite.internal.schema.row.InternalTuple; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +/** Tests to verify projected tuple facade. */ +class ProjectedTupleTest { + private static final IgniteLogger LOG = Loggers.forClass(ProjectedTupleTest.class); + + private static final BinaryTupleSchema ALL_TYPES_SCHEMA = BinaryTupleSchema.create( + SchemaTestUtils.ALL_TYPES.stream() + .map(type -> new Element(type, true)) + .toArray(Element[]::new) + ); + + private static BinaryTuple TUPLE; + private static Random RND; + + @BeforeAll + static void initRandom() { + int seed = ThreadLocalRandom.current().nextInt(); + + RND = new Random(seed); + + LOG.info("Seed is " + seed); + + var builder = new BinaryTupleBuilder(ALL_TYPES_SCHEMA.elementCount()); + + for (int i = 0; i < ALL_TYPES_SCHEMA.elementCount(); i++) { + Element e = ALL_TYPES_SCHEMA.element(i); + + BinaryRowConverter.appendValue(builder, e, SchemaTestUtils.generateRandomValue(RND, fromElement(e))); + } + + TUPLE = new BinaryTuple(ALL_TYPES_SCHEMA.elementCount(), builder.build()); + } + + @Test + void allTypesAreCovered() { + List<NativeTypeSpec> coveredTypes = IntStream.range(0, ALL_TYPES_SCHEMA.elementCount()) + .mapToObj(ALL_TYPES_SCHEMA::element) + .map(Element::typeSpec) + .collect(Collectors.toList()); + + EnumSet<NativeTypeSpec> allTypes = EnumSet.allOf(NativeTypeSpec.class); + + coveredTypes.forEach(allTypes::remove); + + assertThat(allTypes, empty()); + } + + @ParameterizedTest + @ValueSource(ints = {1, 2, 3}) + void projectionReturnsProperElementCount(int projectionSize) { + InternalTuple projection1 = new ProjectedTuple( + ALL_TYPES_SCHEMA, TUPLE, new int[projectionSize] + ); + + assertThat(projection1.elementCount(), equalTo(projectionSize)); + } + + @ParameterizedTest + @ValueSource(booleans = {true, false}) + void testProjection(boolean useOptimizeProjection) { + int f1 = RND.nextInt(ALL_TYPES_SCHEMA.elementCount()); + int f2 = RND.nextInt(ALL_TYPES_SCHEMA.elementCount()); + int f3 = RND.nextInt(ALL_TYPES_SCHEMA.elementCount()); + + int[] projection = {f1, f2, f3}; + + InternalTuple projectedTuple = useOptimizeProjection + ? new ProjectedTuple(TUPLE, projection) + : new ProjectedTuple(ALL_TYPES_SCHEMA, TUPLE, projection); + + Element e1 = ALL_TYPES_SCHEMA.element(f1); + Element e2 = ALL_TYPES_SCHEMA.element(f2); + Element e3 = ALL_TYPES_SCHEMA.element(f3); + + BinaryTupleSchema projectedSchema = BinaryTupleSchema.create(new Element[] { + e1, e2, e3 + }); + + assertThat(projectedSchema.value(projectedTuple, 0), equalTo(ALL_TYPES_SCHEMA.value(TUPLE, f1))); + assertThat(projectedSchema.value(projectedTuple, 1), equalTo(ALL_TYPES_SCHEMA.value(TUPLE, f2))); + assertThat(projectedSchema.value(projectedTuple, 2), equalTo(ALL_TYPES_SCHEMA.value(TUPLE, f3))); + + InternalTuple restored = new BinaryTuple(projection.length, projectedTuple.byteBuffer()); + + assertThat(projectedSchema.value(restored, 0), equalTo(ALL_TYPES_SCHEMA.value(TUPLE, f1))); + assertThat(projectedSchema.value(restored, 1), equalTo(ALL_TYPES_SCHEMA.value(TUPLE, f2))); + assertThat(projectedSchema.value(restored, 2), equalTo(ALL_TYPES_SCHEMA.value(TUPLE, f3))); + } + + private static NativeType fromElement(Element element) { + switch (element.typeSpec()) { Review Comment: I see two other test classes that implement such a method. `AbstractSerializerTest#specToType` and `AbstractSchemaConverterTest#specToType`. Seems (maybe someday in the future :roll_eyes:) we'll need to extract it into a common utility class (`SchemaTestUtils`) but provide ability to pass decimal precision/scale arguments :thinking: ########## modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/util/ProjectedTuple.java: ########## @@ -0,0 +1,322 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.sql.engine.util; + +import java.math.BigDecimal; +import java.math.BigInteger; +import java.nio.ByteBuffer; +import java.time.Instant; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.LocalTime; +import java.util.BitSet; +import java.util.Objects; +import java.util.UUID; +import org.apache.ignite.internal.binarytuple.BinaryTupleBuilder; +import org.apache.ignite.internal.binarytuple.BinaryTupleParser; +import org.apache.ignite.internal.binarytuple.BinaryTupleParser.Sink; +import org.apache.ignite.internal.schema.BinaryRowConverter; +import org.apache.ignite.internal.schema.BinaryTuple; +import org.apache.ignite.internal.schema.BinaryTupleSchema; +import org.apache.ignite.internal.schema.BinaryTupleSchema.Element; +import org.apache.ignite.internal.schema.row.InternalTuple; +import org.jetbrains.annotations.Nullable; + +/** + * A facade that creates projection of the given tuple. + * + * <p>Not thread safe! + * + * <p>This projection is used to change indexes of column in original tuple, or to trim + * few columns from original tuple. Here are a few examples:<pre> + * Having tuple ['foo', 'bar', 'baz'], we can + * + * - reorder fields with mapping [2, 1, 0] to get equivalent tuple ['baz', 'bar', 'foo'] + * - or trim certain fields with mapping [0, 2] to get equivalent tuple ['foo', 'baz'] + * - or even repeat some fields with mapping [0, 0, 0] to get equivalent tuple ['foo', 'foo', 'foo'] + * </pre> + */ +public class ProjectedTuple implements InternalTuple { + private final @Nullable BinaryTupleSchema schema; + + private InternalTuple delegate; + private int[] projection; + + private boolean normalized = false; + + /** + * Creates projected tuple with not optimal but reliable conversion. + * + * <p>When call to {@link #byteBuffer()}, the original tuple will be read field by field with regard to provided projection. + * Such an approach had an additional overhead on (de-)serialization fields value, but had no requirement for the tuple + * to be compatible with Binary Tuple format. + * + * @param schema A schema of the original tuple (represented by delegate). Used to read content of the delegate to build a + * proper byte buffer which content satisfying the schema with regard to given projection. + * @param delegate An original tuple to create projection from. + * @param projection A projection. That is, desired order of fields in original tuple. In that projection, index of the array is + * an index of field in resulting projection, and an element of the array at that index is an index of column in original + * tuple. + */ + public ProjectedTuple( + BinaryTupleSchema schema, + InternalTuple delegate, + int[] projection + ) { + this.schema = Objects.requireNonNull(schema); + this.delegate = delegate; + this.projection = projection; + } + + /** + * Creates projected tuple with optimized conversion. + * + * <p>When call to {@link #byteBuffer()}, the original tuple will be rebuild with regard to provided projection + * by copying raw bytes from original tuple. Although this works more optimal, it requires an original tuple + * to be crafted with regard to Binary Tuple format. + * + * <p>It's up to the caller to get sure that provided tuple respect the format. + * + * @param delegate An original tuple to create projection from. + * @param projection A projection. That is, desired order of fields in original tuple. In that projection, index of the array is + * an index of field in resulting projection, and an element of the array at that index is an index of column in original + * tuple. + */ + public ProjectedTuple( + InternalTuple delegate, + int[] projection + ) { + this.delegate = delegate; + this.projection = projection; + + this.schema = null; + } + + @Override + public int elementCount() { + return projection.length; + } + + @Override + public boolean hasNullValue(int col) { + return delegate.hasNullValue(projection[col]); + } + + @Override + public boolean booleanValue(int col) { + return delegate.booleanValue(projection[col]); + } + + @Override + public Boolean booleanValueBoxed(int col) { + return delegate.booleanValueBoxed(projection[col]); + } + + @Override + public byte byteValue(int col) { + return delegate.byteValue(projection[col]); + } + + @Override + public Byte byteValueBoxed(int col) { + return delegate.byteValueBoxed(projection[col]); + } + + @Override + public short shortValue(int col) { + return delegate.shortValue(projection[col]); + } + + @Override + public Short shortValueBoxed(int col) { + return delegate.shortValueBoxed(projection[col]); + } + + @Override + public int intValue(int col) { + return delegate.intValue(projection[col]); + } + + @Override + public Integer intValueBoxed(int col) { + return delegate.intValueBoxed(projection[col]); + } + + @Override + public long longValue(int col) { + return delegate.longValue(projection[col]); + } + + @Override + public Long longValueBoxed(int col) { + return delegate.longValueBoxed(projection[col]); + } + + @Override + public float floatValue(int col) { + return delegate.floatValue(projection[col]); + } + + @Override + public Float floatValueBoxed(int col) { + return delegate.floatValueBoxed(projection[col]); + } + + @Override + public double doubleValue(int col) { + return delegate.doubleValue(projection[col]); + } + + @Override + public Double doubleValueBoxed(int col) { + return delegate.doubleValueBoxed(projection[col]); + } + + @Override + public BigDecimal decimalValue(int col, int decimalScale) { + return delegate.decimalValue(projection[col], decimalScale); + } + + @Override + public BigInteger numberValue(int col) { + return delegate.numberValue(projection[col]); + } + + @Override + public String stringValue(int col) { + return delegate.stringValue(projection[col]); + } + + @Override + public byte[] bytesValue(int col) { + return delegate.bytesValue(projection[col]); + } + + @Override + public UUID uuidValue(int col) { + return delegate.uuidValue(projection[col]); + } + + @Override + public BitSet bitmaskValue(int col) { + return delegate.bitmaskValue(projection[col]); + } + + @Override + public LocalDate dateValue(int col) { + return delegate.dateValue(projection[col]); + } + + @Override + public LocalTime timeValue(int col) { + return delegate.timeValue(projection[col]); + } + + @Override + public LocalDateTime dateTimeValue(int col) { + return delegate.dateTimeValue(projection[col]); + } + + @Override + public Instant timestampValue(int col) { + return delegate.timestampValue(projection[col]); + } + + @Override + public ByteBuffer byteBuffer() { + normalizeIfNeeded(); + + return delegate.byteBuffer(); + } + + private void normalizeIfNeeded() { + if (normalized) { + return; + } + + if (schema != null) { Review Comment: I'm not insisting, but it seems that the code may be cleaner if we split this into two classes (a base without a schema, and a descendant with a schema). I don't know what to call them beautifully. , we can add a factory method to create a ProjectedTuple, which will create the desired internal implementation. ########## modules/sql-engine/src/test/java/org/apache/ignite/internal/sql/engine/util/ProjectedTupleTest.java: ########## @@ -0,0 +1,175 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.sql.engine.util; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.empty; +import static org.hamcrest.Matchers.equalTo; + +import java.util.EnumSet; +import java.util.List; +import java.util.Random; +import java.util.concurrent.ThreadLocalRandom; +import java.util.stream.Collectors; +import java.util.stream.IntStream; +import org.apache.ignite.internal.binarytuple.BinaryTupleBuilder; +import org.apache.ignite.internal.logger.IgniteLogger; +import org.apache.ignite.internal.logger.Loggers; +import org.apache.ignite.internal.schema.BinaryRowConverter; +import org.apache.ignite.internal.schema.BinaryTuple; +import org.apache.ignite.internal.schema.BinaryTupleSchema; +import org.apache.ignite.internal.schema.BinaryTupleSchema.Element; +import org.apache.ignite.internal.schema.NativeType; +import org.apache.ignite.internal.schema.NativeTypeSpec; +import org.apache.ignite.internal.schema.NativeTypes; +import org.apache.ignite.internal.schema.SchemaTestUtils; +import org.apache.ignite.internal.schema.row.InternalTuple; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +/** Tests to verify projected tuple facade. */ +class ProjectedTupleTest { + private static final IgniteLogger LOG = Loggers.forClass(ProjectedTupleTest.class); + + private static final BinaryTupleSchema ALL_TYPES_SCHEMA = BinaryTupleSchema.create( + SchemaTestUtils.ALL_TYPES.stream() + .map(type -> new Element(type, true)) + .toArray(Element[]::new) + ); + + private static BinaryTuple TUPLE; + private static Random RND; + + @BeforeAll + static void initRandom() { + int seed = ThreadLocalRandom.current().nextInt(); + + RND = new Random(seed); + + LOG.info("Seed is " + seed); + + var builder = new BinaryTupleBuilder(ALL_TYPES_SCHEMA.elementCount()); + + for (int i = 0; i < ALL_TYPES_SCHEMA.elementCount(); i++) { + Element e = ALL_TYPES_SCHEMA.element(i); + + BinaryRowConverter.appendValue(builder, e, SchemaTestUtils.generateRandomValue(RND, fromElement(e))); + } + + TUPLE = new BinaryTuple(ALL_TYPES_SCHEMA.elementCount(), builder.build()); + } + + @Test + void allTypesAreCovered() { + List<NativeTypeSpec> coveredTypes = IntStream.range(0, ALL_TYPES_SCHEMA.elementCount()) + .mapToObj(ALL_TYPES_SCHEMA::element) + .map(Element::typeSpec) + .collect(Collectors.toList()); + + EnumSet<NativeTypeSpec> allTypes = EnumSet.allOf(NativeTypeSpec.class); + + coveredTypes.forEach(allTypes::remove); + + assertThat(allTypes, empty()); + } + + @ParameterizedTest + @ValueSource(ints = {1, 2, 3}) + void projectionReturnsProperElementCount(int projectionSize) { + InternalTuple projection1 = new ProjectedTuple( + ALL_TYPES_SCHEMA, TUPLE, new int[projectionSize] + ); + + assertThat(projection1.elementCount(), equalTo(projectionSize)); + } + + @ParameterizedTest + @ValueSource(booleans = {true, false}) + void testProjection(boolean useOptimizeProjection) { + int f1 = RND.nextInt(ALL_TYPES_SCHEMA.elementCount()); + int f2 = RND.nextInt(ALL_TYPES_SCHEMA.elementCount()); + int f3 = RND.nextInt(ALL_TYPES_SCHEMA.elementCount()); + + int[] projection = {f1, f2, f3}; + + InternalTuple projectedTuple = useOptimizeProjection + ? new ProjectedTuple(TUPLE, projection) + : new ProjectedTuple(ALL_TYPES_SCHEMA, TUPLE, projection); + + Element e1 = ALL_TYPES_SCHEMA.element(f1); + Element e2 = ALL_TYPES_SCHEMA.element(f2); + Element e3 = ALL_TYPES_SCHEMA.element(f3); + + BinaryTupleSchema projectedSchema = BinaryTupleSchema.create(new Element[] { + e1, e2, e3 + }); + + assertThat(projectedSchema.value(projectedTuple, 0), equalTo(ALL_TYPES_SCHEMA.value(TUPLE, f1))); + assertThat(projectedSchema.value(projectedTuple, 1), equalTo(ALL_TYPES_SCHEMA.value(TUPLE, f2))); + assertThat(projectedSchema.value(projectedTuple, 2), equalTo(ALL_TYPES_SCHEMA.value(TUPLE, f3))); + + InternalTuple restored = new BinaryTuple(projection.length, projectedTuple.byteBuffer()); + + assertThat(projectedSchema.value(restored, 0), equalTo(ALL_TYPES_SCHEMA.value(TUPLE, f1))); + assertThat(projectedSchema.value(restored, 1), equalTo(ALL_TYPES_SCHEMA.value(TUPLE, f2))); + assertThat(projectedSchema.value(restored, 2), equalTo(ALL_TYPES_SCHEMA.value(TUPLE, f3))); + } + + private static NativeType fromElement(Element element) { + switch (element.typeSpec()) { + case BOOLEAN: + return NativeTypes.BOOLEAN; + case INT8: + return NativeTypes.INT8; + case INT16: + return NativeTypes.INT16; + case INT32: + return NativeTypes.INT32; + case INT64: + return NativeTypes.INT64; + case FLOAT: + return NativeTypes.FLOAT; + case DOUBLE: + return NativeTypes.DOUBLE; + case DECIMAL: + return NativeTypes.decimalOf(20, element.decimalScale()); + case NUMBER: + return NativeTypes.numberOf(20); + case DATE: + return NativeTypes.DATE; + case TIME: + return NativeTypes.time(); + case DATETIME: + return NativeTypes.datetime(); + case TIMESTAMP: + return NativeTypes.timestamp(); + case UUID: + return NativeTypes.UUID; + case BITMASK: + return NativeTypes.bitmaskOf(256); + case STRING: + return NativeTypes.stringOf(256); + case BYTES: + return NativeTypes.blobOf(256); + default: + throw new IllegalArgumentException("Unknown type: " + element.typeSpec()); + } + } +} Review Comment: We usually put an empty line at the end of the file. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
