JingsongLi commented on a change in pull request #7816: [FLINK-11701][table-runtime-blink] Introduce an abstract set of data formats URL: https://github.com/apache/flink/pull/7816#discussion_r261471313
########## File path: flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/dataformat/BinaryRow.java ########## @@ -0,0 +1,289 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.dataformat; + +import org.apache.flink.core.memory.MemorySegment; +import org.apache.flink.core.memory.MemorySegmentFactory; +import org.apache.flink.table.util.SegmentsUtil; + +import java.nio.ByteOrder; + +import static org.apache.flink.util.Preconditions.checkArgument; + +/** + * A special row which is backed by {@link MemorySegment} instead of Object. It can significantly + * reduce the serialization/deserialization of Java objects. + * + * <p>A Row has two part: Fixed-length part and variable-length part. + * + * <p>Fixed-length part contains 1 byte header and null bit set and field values. Null bit set is + * used for null tracking and is aligned to 8-byte word boundaries. `Field values` holds + * fixed-length primitive types and variable-length values which can be stored in 8 bytes inside. + * If it do not fit the variable-length field, then store the length and offset of variable-length + * part. + * + * <p>Fixed-length part will certainly fall into a MemorySegment, which will speed up the read + * and write of field. During the write phase, if the target memory segment has less space than + * fixed length part size, we will skip the space. So the number of fields in a single Row cannot + * exceed the capacity of a single MemorySegment, if there are too many fields, we suggest that + * user set a bigger pageSize of MemorySegment. + * + * <p>Variable-length part may fall into multiple MemorySegments. + * + * <p>{@code BinaryRow} are influenced by Apache Spark UnsafeRow in project tungsten. + * The difference is that BinaryRow is placed on a discontinuous memory, and the variable length + * type can also be placed on a fixed length area (If it's short enough). + */ +public final class BinaryRow extends BinaryFormat<Object> implements BaseRow { + + public static final boolean LITTLE_ENDIAN = (ByteOrder.nativeOrder() == ByteOrder.LITTLE_ENDIAN); + public static final long FIRST_BYTE_ZERO = LITTLE_ENDIAN ? 0xFFF0 : 0x0FFF; + + public static int calculateBitSetWidthInBytes(int arity) { + // add 8 bit header + return ((arity + 63 + 8) / 64) * 8; + } + + private final int arity; + private final int nullBitsSizeInBytes; + + public BinaryRow(int arity) { + checkArgument(arity >= 0); + this.arity = arity; + this.nullBitsSizeInBytes = calculateBitSetWidthInBytes(arity); + } + + private int getFieldOffset(int pos) { + return offset + nullBitsSizeInBytes + pos * 8; + } + + private void assertIndexIsValid(int index) { + assert index >= 0 : "index (" + index + ") should >= 0"; + assert index < arity : "index (" + index + ") should < " + arity; + } + + public int getFixedLengthPartSize() { + return nullBitsSizeInBytes + 8 * arity; + } + + @Override + public int getArity() { + return arity; + } + + @Override + public byte getHeader() { + // first nullBitsSizeInBytes byte is header. + return segments[0].get(offset); + } + + @Override + public void setHeader(byte header) { + segments[0].put(offset, header); + } + + public void pointTo(MemorySegment segment, int offset, int sizeInBytes) { + this.segments = new MemorySegment[] {segment}; + this.offset = offset; + this.sizeInBytes = sizeInBytes; + } + + public void pointTo(MemorySegment[] segments, int offset, int sizeInBytes) { + this.segments = segments; + this.offset = offset; + this.sizeInBytes = sizeInBytes; + } + + public void setTotalSize(int sizeInBytes) { + this.sizeInBytes = sizeInBytes; + } + + private void setNotNullAt(int i) { + assertIndexIsValid(i); + // need add header 8 bit. + SegmentsUtil.bitUnSet(segments[0], offset, i + 8); Review comment: OK ---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: [email protected] With regards, Apache Git Services
