sunchao commented on code in PR #36616:
URL: https://github.com/apache/spark/pull/36616#discussion_r900538256
##########
sql/core/src/main/java/org/apache/spark/sql/execution/vectorized/ColumnVectorUtils.java:
##########
@@ -105,6 +106,73 @@ public static void populate(WritableColumnVector col,
InternalRow row, int field
}
}
+ /**
+ * Fill value of `row[fieldIdx]` into `ConstantColumnVector`.
+ */
+ public static void fill(ConstantColumnVector col, InternalRow row, int
fieldIdx) {
Review Comment:
can we call this `populate` to keep it consistent? in fact we can probably
remove the usage of the other `populate` method and replace it with this one
##########
sql/core/src/main/java/org/apache/spark/sql/execution/vectorized/ColumnVectorUtils.java:
##########
@@ -235,4 +303,37 @@ public static ColumnarBatch toBatch(
batch.setNumRows(n);
return batch;
}
+
+ /**
+ * <b>This method assumes that all constant column are at the end of schema
+ * and `constantColumnLength` represents the number of constant column.<b/>
+ *
+ * This method allocates columns to store elements of each field of the
schema,
+ * the data columns use `OffHeapColumnVector` when `useOffHeap` is true and
+ * use `OnHeapColumnVector` when `useOffHeap` is false, the constant columns
+ * always use `ConstantColumnVector`.
+ *
+ * Capacity is the initial capacity of the vector, and it will grow as
necessary.
+ * Capacity is in number of elements, not number of bytes.
+ */
+ public static ColumnVector[] allocateColumns(
Review Comment:
hmm I wonder why we need this method as there is only one call site for it
in `VectorizedParquetRecordReader`.
##########
sql/core/src/main/java/org/apache/spark/sql/execution/vectorized/ColumnVectorUtils.java:
##########
@@ -105,6 +106,73 @@ public static void populate(WritableColumnVector col,
InternalRow row, int field
}
}
+ /**
+ * Fill value of `row[fieldIdx]` into `ConstantColumnVector`.
+ */
+ public static void fill(ConstantColumnVector col, InternalRow row, int
fieldIdx) {
+ DataType t = col.dataType();
+
+ if (row.isNullAt(fieldIdx)) {
+ col.setNull();
+ } else {
+ if (t == DataTypes.BooleanType) {
+ col.setBoolean(row.getBoolean(fieldIdx));
+ } else if (t == DataTypes.BinaryType) {
+ col.setBinary(row.getBinary(fieldIdx));
+ } else if (t == DataTypes.ByteType) {
+ col.setByte(row.getByte(fieldIdx));
+ } else if (t == DataTypes.ShortType) {
+ col.setShort(row.getShort(fieldIdx));
+ } else if (t == DataTypes.IntegerType) {
+ col.setInt(row.getInt(fieldIdx));
+ } else if (t == DataTypes.LongType) {
+ col.setLong(row.getLong(fieldIdx));
+ } else if (t == DataTypes.FloatType) {
+ col.setFloat(row.getFloat(fieldIdx));
+ } else if (t == DataTypes.DoubleType) {
+ col.setDouble(row.getDouble(fieldIdx));
+ } else if (t == DataTypes.StringType) {
+ UTF8String v = row.getUTF8String(fieldIdx);
+ col.setUtf8String(v);
+ } else if (t instanceof DecimalType) {
+ DecimalType dt = (DecimalType)t;
+ Decimal d = row.getDecimal(fieldIdx, dt.precision(), dt.scale());
+ if (dt.precision() <= Decimal.MAX_INT_DIGITS()) {
+ col.setInt((int)d.toUnscaledLong());
+ } else if (dt.precision() <= Decimal.MAX_LONG_DIGITS()) {
+ col.setLong(d.toUnscaledLong());
+ } else {
+ final BigInteger integer = d.toJavaBigDecimal().unscaledValue();
+ byte[] bytes = integer.toByteArray();
+ col.setBinary(bytes);
+ }
+ } else if (t instanceof CalendarIntervalType) {
+ CalendarInterval c = (CalendarInterval)row.get(fieldIdx, t);
+ // The value of `numRows` is irrelevant.
+ ConstantColumnVector monthsVector =
Review Comment:
I wonder if we should just initialize the child columns in the
`ConstantColumnVector` ctor if its data type is `CalendarIntervalType`.
Otherwise, we'd create 3 child vectors for **each `fill` call** on the same
vector (even though it's unlikely used that way?).
##########
sql/core/src/main/java/org/apache/spark/sql/execution/vectorized/ColumnVectorUtils.java:
##########
@@ -105,6 +106,73 @@ public static void populate(WritableColumnVector col,
InternalRow row, int field
}
}
+ /**
+ * Fill value of `row[fieldIdx]` into `ConstantColumnVector`.
+ */
+ public static void fill(ConstantColumnVector col, InternalRow row, int
fieldIdx) {
+ DataType t = col.dataType();
+
+ if (row.isNullAt(fieldIdx)) {
+ col.setNull();
+ } else {
+ if (t == DataTypes.BooleanType) {
+ col.setBoolean(row.getBoolean(fieldIdx));
+ } else if (t == DataTypes.BinaryType) {
+ col.setBinary(row.getBinary(fieldIdx));
+ } else if (t == DataTypes.ByteType) {
+ col.setByte(row.getByte(fieldIdx));
+ } else if (t == DataTypes.ShortType) {
+ col.setShort(row.getShort(fieldIdx));
+ } else if (t == DataTypes.IntegerType) {
+ col.setInt(row.getInt(fieldIdx));
+ } else if (t == DataTypes.LongType) {
+ col.setLong(row.getLong(fieldIdx));
+ } else if (t == DataTypes.FloatType) {
+ col.setFloat(row.getFloat(fieldIdx));
+ } else if (t == DataTypes.DoubleType) {
+ col.setDouble(row.getDouble(fieldIdx));
+ } else if (t == DataTypes.StringType) {
+ UTF8String v = row.getUTF8String(fieldIdx);
+ col.setUtf8String(v);
+ } else if (t instanceof DecimalType) {
+ DecimalType dt = (DecimalType)t;
+ Decimal d = row.getDecimal(fieldIdx, dt.precision(), dt.scale());
+ if (dt.precision() <= Decimal.MAX_INT_DIGITS()) {
+ col.setInt((int)d.toUnscaledLong());
+ } else if (dt.precision() <= Decimal.MAX_LONG_DIGITS()) {
+ col.setLong(d.toUnscaledLong());
+ } else {
+ final BigInteger integer = d.toJavaBigDecimal().unscaledValue();
+ byte[] bytes = integer.toByteArray();
+ col.setBinary(bytes);
+ }
+ } else if (t instanceof CalendarIntervalType) {
+ CalendarInterval c = (CalendarInterval)row.get(fieldIdx, t);
+ // The value of `numRows` is irrelevant.
+ ConstantColumnVector monthsVector =
+ new ConstantColumnVector(1, IntegerType$.MODULE$);
Review Comment:
we can use `DataTypes.IntegerType` here
##########
sql/core/src/main/java/org/apache/spark/sql/execution/vectorized/ColumnVectorUtils.java:
##########
@@ -105,6 +106,73 @@ public static void populate(WritableColumnVector col,
InternalRow row, int field
}
}
+ /**
+ * Fill value of `row[fieldIdx]` into `ConstantColumnVector`.
+ */
+ public static void fill(ConstantColumnVector col, InternalRow row, int
fieldIdx) {
+ DataType t = col.dataType();
+
+ if (row.isNullAt(fieldIdx)) {
+ col.setNull();
+ } else {
+ if (t == DataTypes.BooleanType) {
+ col.setBoolean(row.getBoolean(fieldIdx));
+ } else if (t == DataTypes.BinaryType) {
+ col.setBinary(row.getBinary(fieldIdx));
+ } else if (t == DataTypes.ByteType) {
+ col.setByte(row.getByte(fieldIdx));
+ } else if (t == DataTypes.ShortType) {
+ col.setShort(row.getShort(fieldIdx));
+ } else if (t == DataTypes.IntegerType) {
+ col.setInt(row.getInt(fieldIdx));
+ } else if (t == DataTypes.LongType) {
+ col.setLong(row.getLong(fieldIdx));
+ } else if (t == DataTypes.FloatType) {
+ col.setFloat(row.getFloat(fieldIdx));
+ } else if (t == DataTypes.DoubleType) {
+ col.setDouble(row.getDouble(fieldIdx));
+ } else if (t == DataTypes.StringType) {
+ UTF8String v = row.getUTF8String(fieldIdx);
+ col.setUtf8String(v);
+ } else if (t instanceof DecimalType) {
+ DecimalType dt = (DecimalType)t;
Review Comment:
nit: can we leave a space after the cast `(Decimal) t`? same for other places
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
