GitHub user kiszk opened a pull request:
https://github.com/apache/spark/pull/13663
Spark SparkSPARK-15950 Eliminate unreachable code at projection for
complex types
## What changes were proposed in this pull request?
This PR eliminates unreachable code at projection for complex types (e.g.
array, map, and struct) to reduce the total size of Java byte code in a method
generated by whole stage codegen.
This PR eliminates the following code blocks in the following cases:
1. A type of ```project_value``` is ```UnsafeArrayData```
2. ```project_value is``` ```null```
3. each element of ```project_values``` is ```null``` if it never occurs
Examples for complex types
````java
val df = sparkContext.parallelize(Seq(1, 2), 1).toDF("v")
df.selectExpr("Array(v + 2, v + 3)").collect
df.selectExpr("struct(v + 3, v + 4)").collect
df.selectExpr("map(v + 3, v + 4)").collect
````
Before applying this PR into ```Array```
```java
/* 028 */ protected void processNext() throws java.io.IOException {
/* 029 */ while (inputadapter_input.hasNext()) {
/* 030 */ InternalRow inputadapter_row = (InternalRow)
inputadapter_input.next();
/* 031 */ final boolean project_isNull = false;
/* 032 */ this.project_values = new Object[2];
/* 033 */ double inputadapter_value = inputadapter_row.getDouble(0);
/* 034 */
/* 035 */ double project_value1 = -1.0;
/* 036 */ project_value1 = inputadapter_value + 2.2D;
/* 037 */ if (false) {
/* 038 */ project_values[0] = null;
/* 039 */ } else {
/* 040 */ project_values[0] = project_value1;
/* 041 */ }
/* 042 */
/* 043 */ double inputadapter_value1 = inputadapter_row.getDouble(1);
/* 044 */
/* 045 */ double project_value4 = -1.0;
/* 046 */ project_value4 = inputadapter_value1 + 3.3D;
/* 047 */ if (false) {
/* 048 */ project_values[1] = null;
/* 049 */ } else {
/* 050 */ project_values[1] = project_value4;
/* 051 */ }
/* 052 */
/* 053 */ final ArrayData project_value = new
org.apache.spark.sql.catalyst.util.GenericArrayData(project_values);
/* 054 */ this.project_values = null;
/* 055 */ project_holder.reset();
/* 056 */
/* 057 */ project_rowWriter.zeroOutNullBytes();
/* 058 */
/* 059 */ if (project_isNull) {
/* 060 */ project_rowWriter.setNullAt(0);
/* 061 */ } else {
/* 062 */ // Remember the current cursor so that we can calculate
how many bytes are
/* 063 */ // written later.
/* 064 */ final int project_tmpCursor = project_holder.cursor;
/* 065 */
/* 066 */ if (project_value instanceof UnsafeArrayData) {
/* 067 */ final int project_sizeInBytes = ((UnsafeArrayData)
project_value).getSizeInBytes();
/* 068 */ // grow the global buffer before writing data.
/* 069 */ project_holder.grow(project_sizeInBytes);
/* 070 */ ((UnsafeArrayData)
project_value).writeToMemory(project_holder.buffer, project_holder.cursor);
/* 071 */ project_holder.cursor += project_sizeInBytes;
/* 072 */
/* 073 */ } else {
/* 074 */ final int project_numElements =
project_value.numElements();
/* 075 */ project_arrayWriter.initialize(project_holder,
project_numElements, 8);
/* 076 */
/* 077 */ for (int project_index = 0; project_index <
project_numElements; project_index++) {
/* 078 */ if (project_value.isNullAt(project_index)) {
/* 079 */ project_arrayWriter.setNullAt(project_index);
/* 080 */ } else {
/* 081 */ final double project_element =
project_value.getDouble(project_index);
/* 082 */ project_arrayWriter.write(project_index,
project_element);
/* 083 */ }
/* 084 */ }
/* 085 */ }
/* 086 */
/* 087 */ project_rowWriter.setOffsetAndSize(0, project_tmpCursor,
project_holder.cursor - project_tmpCursor);
/* 088 */ project_rowWriter.alignToWords(project_holder.cursor -
project_tmpCursor);
/* 089 */ }
/* 090 */ project_result.setTotalSize(project_holder.totalSize());
/* 091 */ append(project_result);
/* 092 */ if (shouldStop()) return;
/* 093 */ }
/* 094 */ }
````
After applying this PR into ```Array```
````java
/* 028 */ protected void processNext() throws java.io.IOException {
/* 029 */ while (inputadapter_input.hasNext()) {
/* 030 */ InternalRow inputadapter_row = (InternalRow)
inputadapter_input.next();
/* 031 */ final boolean project_isNull = false;
/* 032 */ this.project_values = new Object[2];
/* 033 */ double inputadapter_value = inputadapter_row.getDouble(0);
/* 034 */
/* 035 */ double project_value1 = -1.0;
/* 036 */ project_value1 = inputadapter_value + 2.2D;
/* 037 */ project_values[0] = project_value1;
/* 038 */ double inputadapter_value1 = inputadapter_row.getDouble(1);
/* 039 */
/* 040 */ double project_value4 = -1.0;
/* 041 */ project_value4 = inputadapter_value1 + 3.3D;
/* 042 */ project_values[1] = project_value4;
/* 043 */ final ArrayData project_value = new
org.apache.spark.sql.catalyst.util.GenericArrayData(project_values);
/* 044 */ this.project_values = null;
/* 045 */ project_holder.reset();
/* 046 */
/* 047 */ // Remember the current cursor so that we can calculate how
many bytes are
/* 048 */ // written later.
/* 049 */ final int project_tmpCursor = project_holder.cursor;
/* 050 */
/* 051 */ final int project_numElements = project_value.numElements();
/* 052 */ project_arrayWriter.initialize(project_holder,
project_numElements, 8);
/* 053 */
/* 054 */ for (int project_index = 0; project_index <
project_numElements; project_index++) {
/* 055 */ if (project_value.isNullAt(project_index)) {
/* 056 */ project_arrayWriter.setNullAt(project_index);
/* 057 */ } else {
/* 058 */ final double project_element =
project_value.getDouble(project_index);
/* 059 */ project_arrayWriter.write(project_index,
project_element);
/* 060 */ }
/* 061 */ }
/* 062 */
/* 063 */ project_rowWriter.setOffsetAndSize(0, project_tmpCursor,
project_holder.cursor - project_tmpCursor);
/* 064 */ project_rowWriter.alignToWords(project_holder.cursor -
project_tmpCursor);
/* 065 */ project_result.setTotalSize(project_holder.totalSize());
/* 066 */ append(project_result);
/* 067 */ if (shouldStop()) return;
/* 068 */ }
/* 069 */ }
````
## How was this patch tested?
Added some unit tests into ```DataFrameComplexTypeSuite```
You can merge this pull request into a Git repository by running:
$ git pull https://github.com/kiszk/spark SPARK-15950
Alternatively you can review and apply these changes as the patch at:
https://github.com/apache/spark/pull/13663.patch
To close this pull request, make a commit to your master/trunk branch
with (at least) the following in the commit message:
This closes #13663
----
----
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
