Tartarus0zm commented on code in PR #2063: URL: https://github.com/apache/auron/pull/2063#discussion_r2889574287
########## auron-flink-extension/auron-flink-runtime/src/main/java/org/apache/auron/flink/arrow/vectors/ArrowBigIntColumnVector.java: ########## @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.auron.flink.arrow.vectors; + +import org.apache.arrow.vector.BigIntVector; +import org.apache.flink.table.data.columnar.vector.LongColumnVector; +import org.apache.flink.util.Preconditions; + +/** + * A Flink {@link LongColumnVector} backed by an Arrow {@link BigIntVector}. + * + * <p>This wrapper delegates all reads to the underlying Arrow vector, providing zero-copy access + * to Arrow data from Flink's columnar batch execution engine. + */ +public final class ArrowBigIntColumnVector implements LongColumnVector { + + private BigIntVector vector; + + /** + * Creates a new wrapper around the given Arrow {@link BigIntVector}. + * + * @param vector the Arrow vector to wrap, must not be null + */ + public ArrowBigIntColumnVector(BigIntVector vector) { + this.vector = Preconditions.checkNotNull(vector); + } + + /** {@inheritDoc} */ + @Override + public boolean isNullAt(int i) { + return vector.isNull(i); + } + + /** {@inheritDoc} */ + @Override + public long getLong(int i) { + return vector.get(i); + } + + /** + * Replaces the underlying Arrow vector. Used during reader reset to point at a new batch + * without allocating a new wrapper. + * + * @param vector the new Arrow vector, must not be null + */ + void setVector(BigIntVector vector) { Review Comment: Recommend removing the setVector method 1. Simplifies the code 2. setVector merely eliminates a small amount of overhead from the wrapper class ########## auron-flink-extension/auron-flink-runtime/src/main/java/org/apache/auron/flink/arrow/vectors/ArrowBooleanColumnVector.java: ########## @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.auron.flink.arrow.vectors; + +import org.apache.arrow.vector.BitVector; +import org.apache.flink.table.data.columnar.vector.BooleanColumnVector; +import org.apache.flink.util.Preconditions; + +/** + * A Flink {@link BooleanColumnVector} backed by an Arrow {@link BitVector}. + * + * <p>This wrapper delegates all reads to the underlying Arrow vector, providing zero-copy access + * to Arrow data from Flink's columnar batch execution engine. + */ +public final class ArrowBooleanColumnVector implements BooleanColumnVector { + + private BitVector vector; + + /** + * Creates a new wrapper around the given Arrow {@link BitVector}. + * + * @param vector the Arrow vector to wrap, must not be null + */ + public ArrowBooleanColumnVector(BitVector vector) { + this.vector = Preconditions.checkNotNull(vector); + } + + /** {@inheritDoc} */ + @Override + public boolean isNullAt(int i) { + return vector.isNull(i); + } + + /** {@inheritDoc} */ + @Override + public boolean getBoolean(int i) { + return vector.get(i) != 0; + } + + /** + * Replaces the underlying Arrow vector. Used during reader reset to point at a new batch + * without allocating a new wrapper. + * + * @param vector the new Arrow vector, must not be null + */ + void setVector(BitVector vector) { Review Comment: ditto ########## auron-flink-extension/auron-flink-runtime/src/main/java/org/apache/auron/flink/arrow/vectors/ArrowDoubleColumnVector.java: ########## @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.auron.flink.arrow.vectors; + +import org.apache.arrow.vector.Float8Vector; +import org.apache.flink.table.data.columnar.vector.DoubleColumnVector; +import org.apache.flink.util.Preconditions; + +/** + * A Flink {@link DoubleColumnVector} backed by an Arrow {@link Float8Vector}. + * + * <p>This wrapper delegates all reads to the underlying Arrow vector, providing zero-copy access + * to Arrow data from Flink's columnar batch execution engine. + */ +public final class ArrowDoubleColumnVector implements DoubleColumnVector { + + private Float8Vector vector; + + /** + * Creates a new wrapper around the given Arrow {@link Float8Vector}. + * + * @param vector the Arrow vector to wrap, must not be null + */ + public ArrowDoubleColumnVector(Float8Vector vector) { + this.vector = Preconditions.checkNotNull(vector); + } + + /** {@inheritDoc} */ + @Override + public boolean isNullAt(int i) { + return vector.isNull(i); + } + + /** {@inheritDoc} */ + @Override + public double getDouble(int i) { + return vector.get(i); + } + + /** + * Replaces the underlying Arrow vector. Used during reader reset to point at a new batch + * without allocating a new wrapper. + * + * @param vector the new Arrow vector, must not be null + */ + void setVector(Float8Vector vector) { Review Comment: ditto ########## auron-flink-extension/auron-flink-runtime/src/main/java/org/apache/auron/flink/arrow/vectors/ArrowFloatColumnVector.java: ########## @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.auron.flink.arrow.vectors; + +import org.apache.arrow.vector.Float4Vector; +import org.apache.flink.table.data.columnar.vector.FloatColumnVector; +import org.apache.flink.util.Preconditions; + +/** + * A Flink {@link FloatColumnVector} backed by an Arrow {@link Float4Vector}. + * + * <p>This wrapper delegates all reads to the underlying Arrow vector, providing zero-copy access + * to Arrow data from Flink's columnar batch execution engine. + */ +public final class ArrowFloatColumnVector implements FloatColumnVector { + + private Float4Vector vector; + + /** + * Creates a new wrapper around the given Arrow {@link Float4Vector}. + * + * @param vector the Arrow vector to wrap, must not be null + */ + public ArrowFloatColumnVector(Float4Vector vector) { + this.vector = Preconditions.checkNotNull(vector); + } + + /** {@inheritDoc} */ + @Override + public boolean isNullAt(int i) { + return vector.isNull(i); + } + + /** {@inheritDoc} */ + @Override + public float getFloat(int i) { + return vector.get(i); + } + + /** + * Replaces the underlying Arrow vector. Used during reader reset to point at a new batch + * without allocating a new wrapper. + * + * @param vector the new Arrow vector, must not be null + */ + void setVector(Float4Vector vector) { Review Comment: ditto ########## auron-flink-extension/auron-flink-runtime/src/main/java/org/apache/auron/flink/arrow/vectors/ArrowTimestampColumnVector.java: ########## @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.auron.flink.arrow.vectors; + +import org.apache.arrow.vector.TimeStampVector; +import org.apache.flink.table.data.TimestampData; +import org.apache.flink.table.data.columnar.vector.TimestampColumnVector; +import org.apache.flink.util.Preconditions; + +/** + * A Flink {@link TimestampColumnVector} backed by an Arrow {@link TimeStampVector}. + * + * <p>This wrapper delegates all reads to the underlying Arrow vector, providing zero-copy access + * to Arrow data from Flink's columnar batch execution engine. It handles both {@code + * TimeStampMicroVector} (TIMESTAMP) and {@code TimeStampMicroTZVector} (TIMESTAMP_LTZ) by + * accepting their common parent type {@link TimeStampVector}. Microsecond values are converted to + * Flink's {@link TimestampData} representation (epoch millis + sub-millisecond nanos). + */ +public final class ArrowTimestampColumnVector implements TimestampColumnVector { + + private TimeStampVector vector; + + /** + * Creates a new wrapper around the given Arrow {@link TimeStampVector}. + * + * <p>Accepts both {@code TimeStampMicroVector} and {@code TimeStampMicroTZVector} since they + * share the same storage format and parent type. + * + * @param vector the Arrow vector to wrap, must not be null + */ + public ArrowTimestampColumnVector(TimeStampVector vector) { Review Comment: ditto ########## auron-flink-extension/auron-flink-runtime/src/main/java/org/apache/auron/flink/arrow/vectors/ArrowBooleanColumnVector.java: ########## @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.auron.flink.arrow.vectors; + +import org.apache.arrow.vector.BitVector; +import org.apache.flink.table.data.columnar.vector.BooleanColumnVector; +import org.apache.flink.util.Preconditions; + +/** + * A Flink {@link BooleanColumnVector} backed by an Arrow {@link BitVector}. + * + * <p>This wrapper delegates all reads to the underlying Arrow vector, providing zero-copy access + * to Arrow data from Flink's columnar batch execution engine. + */ +public final class ArrowBooleanColumnVector implements BooleanColumnVector { + + private BitVector vector; + + /** + * Creates a new wrapper around the given Arrow {@link BitVector}. + * + * @param vector the Arrow vector to wrap, must not be null + */ + public ArrowBooleanColumnVector(BitVector vector) { + this.vector = Preconditions.checkNotNull(vector); + } + + /** {@inheritDoc} */ + @Override + public boolean isNullAt(int i) { + return vector.isNull(i); + } + + /** {@inheritDoc} */ + @Override + public boolean getBoolean(int i) { + return vector.get(i) != 0; + } + + /** + * Replaces the underlying Arrow vector. Used during reader reset to point at a new batch + * without allocating a new wrapper. + * + * @param vector the new Arrow vector, must not be null + */ + void setVector(BitVector vector) { + this.vector = Preconditions.checkNotNull(vector); + } Review Comment: I agree with `Copilot` ########## auron-flink-extension/auron-flink-runtime/src/main/java/org/apache/auron/flink/arrow/vectors/ArrowSmallIntColumnVector.java: ########## @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.auron.flink.arrow.vectors; + +import org.apache.arrow.vector.SmallIntVector; +import org.apache.flink.table.data.columnar.vector.ShortColumnVector; +import org.apache.flink.util.Preconditions; + +/** + * A Flink {@link ShortColumnVector} backed by an Arrow {@link SmallIntVector}. + * + * <p>This wrapper delegates all reads to the underlying Arrow vector, providing zero-copy access + * to Arrow data from Flink's columnar batch execution engine. + */ +public final class ArrowSmallIntColumnVector implements ShortColumnVector { + + private SmallIntVector vector; + + /** + * Creates a new wrapper around the given Arrow {@link SmallIntVector}. + * + * @param vector the Arrow vector to wrap, must not be null + */ + public ArrowSmallIntColumnVector(SmallIntVector vector) { + this.vector = Preconditions.checkNotNull(vector); + } + + /** {@inheritDoc} */ + @Override + public boolean isNullAt(int i) { + return vector.isNull(i); + } + + /** {@inheritDoc} */ + @Override + public short getShort(int i) { + return vector.get(i); + } + + /** + * Replaces the underlying Arrow vector. Used during reader reset to point at a new batch + * without allocating a new wrapper. + * + * @param vector the new Arrow vector, must not be null + */ + void setVector(SmallIntVector vector) { Review Comment: ditto ########## auron-flink-extension/auron-flink-runtime/src/main/java/org/apache/auron/flink/arrow/vectors/ArrowTinyIntColumnVector.java: ########## @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.auron.flink.arrow.vectors; + +import org.apache.arrow.vector.TinyIntVector; +import org.apache.flink.table.data.columnar.vector.ByteColumnVector; +import org.apache.flink.util.Preconditions; + +/** + * A Flink {@link ByteColumnVector} backed by an Arrow {@link TinyIntVector}. + * + * <p>This wrapper delegates all reads to the underlying Arrow vector, providing zero-copy access + * to Arrow data from Flink's columnar batch execution engine. + */ +public final class ArrowTinyIntColumnVector implements ByteColumnVector { + + private TinyIntVector vector; + + /** + * Creates a new wrapper around the given Arrow {@link TinyIntVector}. + * + * @param vector the Arrow vector to wrap, must not be null + */ + public ArrowTinyIntColumnVector(TinyIntVector vector) { + this.vector = Preconditions.checkNotNull(vector); + } + + /** {@inheritDoc} */ + @Override + public boolean isNullAt(int i) { + return vector.isNull(i); + } + + /** {@inheritDoc} */ + @Override + public byte getByte(int i) { + return vector.get(i); + } + + /** + * Replaces the underlying Arrow vector. Used during reader reset to point at a new batch + * without allocating a new wrapper. + * + * @param vector the new Arrow vector, must not be null + */ + void setVector(TinyIntVector vector) { Review Comment: ditto ########## auron-flink-extension/auron-flink-runtime/src/main/java/org/apache/auron/flink/arrow/vectors/ArrowTimeColumnVector.java: ########## @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.auron.flink.arrow.vectors; + +import org.apache.arrow.vector.TimeMicroVector; +import org.apache.flink.table.data.columnar.vector.IntColumnVector; +import org.apache.flink.util.Preconditions; + +/** + * A Flink {@link IntColumnVector} backed by an Arrow {@link TimeMicroVector}. + * + * <p>This wrapper delegates all reads to the underlying Arrow vector, providing zero-copy access + * to Arrow data from Flink's columnar batch execution engine. {@link TimeMicroVector} stores time + * values as microseconds since midnight ({@code long}), which are converted to milliseconds + * ({@code int}) to match Flink's internal TIME representation. This reverses the writer's + * conversion from milliseconds to microseconds. + */ +public final class ArrowTimeColumnVector implements IntColumnVector { + + private TimeMicroVector vector; + + /** + * Creates a new wrapper around the given Arrow {@link TimeMicroVector}. + * + * @param vector the Arrow vector to wrap, must not be null + */ + public ArrowTimeColumnVector(TimeMicroVector vector) { Review Comment: why is TimeMicroVector? #1930, I must have overlooked these details. RowData2Arrow is write to Auron. Arrow2RowData is read from Auron. So should we use Auron's current implementation as the standard? ########## auron-flink-extension/auron-flink-runtime/src/main/java/org/apache/auron/flink/arrow/vectors/ArrowDecimalColumnVector.java: ########## @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.auron.flink.arrow.vectors; + +import org.apache.arrow.memory.ArrowBuf; +import org.apache.arrow.vector.DecimalVector; +import org.apache.flink.table.data.DecimalData; +import org.apache.flink.table.data.columnar.vector.DecimalColumnVector; +import org.apache.flink.util.Preconditions; + +/** + * A Flink {@link DecimalColumnVector} backed by an Arrow {@link DecimalVector}. + * + * <p>This wrapper delegates all reads to the underlying Arrow vector, providing zero-copy access + * to Arrow data from Flink's columnar batch execution engine. + * + * <p>For compact decimals (precision <= 18), the implementation avoids BigDecimal allocation by + * reading the unscaled value directly from Arrow's little-endian byte representation. + */ +public final class ArrowDecimalColumnVector implements DecimalColumnVector { + + /** + * Maximum precision that fits in a long (same as Flink's internal compact decimal threshold). + * DecimalData.MAX_COMPACT_PRECISION is package-private, so we define our own constant. + */ + private static final int MAX_COMPACT_PRECISION = 18; + + private DecimalVector vector; + + /** + * Creates a new wrapper around the given Arrow {@link DecimalVector}. + * + * @param vector the Arrow vector to wrap, must not be null + */ + public ArrowDecimalColumnVector(DecimalVector vector) { + this.vector = Preconditions.checkNotNull(vector); + } + + /** {@inheritDoc} */ + @Override + public boolean isNullAt(int i) { + return vector.isNull(i); + } + + /** {@inheritDoc} */ + @Override + public DecimalData getDecimal(int i, int precision, int scale) { + if (precision <= MAX_COMPACT_PRECISION) { Review Comment: I'm not entirely clear on how Arrow handles Decimal types, so we can leave this unchanged for now. Once Arrow2RowData and RowData2Arrow are completed, we can use unit tests to validate read/write operations. ########## auron-flink-extension/auron-flink-runtime/src/main/java/org/apache/auron/flink/arrow/vectors/ArrowIntColumnVector.java: ########## @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.auron.flink.arrow.vectors; + +import org.apache.arrow.vector.IntVector; +import org.apache.flink.table.data.columnar.vector.IntColumnVector; +import org.apache.flink.util.Preconditions; + +/** + * A Flink {@link IntColumnVector} backed by an Arrow {@link IntVector}. + * + * <p>This wrapper delegates all reads to the underlying Arrow vector, providing zero-copy access + * to Arrow data from Flink's columnar batch execution engine. + */ +public final class ArrowIntColumnVector implements IntColumnVector { + + private IntVector vector; + + /** + * Creates a new wrapper around the given Arrow {@link IntVector}. + * + * @param vector the Arrow vector to wrap, must not be null + */ + public ArrowIntColumnVector(IntVector vector) { + this.vector = Preconditions.checkNotNull(vector); + } + + /** {@inheritDoc} */ + @Override + public boolean isNullAt(int i) { + return vector.isNull(i); + } + + /** {@inheritDoc} */ + @Override + public int getInt(int i) { + return vector.get(i); + } + + /** + * Replaces the underlying Arrow vector. Used during reader reset to point at a new batch + * without allocating a new wrapper. + * + * @param vector the new Arrow vector, must not be null + */ + void setVector(IntVector vector) { Review Comment: ditto -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
