GitHub user kiszk opened a pull request:
https://github.com/apache/spark/pull/13243
[SPARK-15258][SQL] Nested/Chained case statements generate codegen over 64k
exception
## What changes were proposed in this pull request?
This PR avoid code generation for SafeProjection if the if the total number
of NewInstances whose code has been generated in
```SpecificSafeProjection.apply()``` exceeds a pre-defined threshold (400). If
the total number exceeds the threshold, ```CodegenFallback.doGenCode``` is
called. This feature avoids an error that bytecode size is larger than 64KB.
Generated code by this PR
````java
/* 001 */
/* 002 */ public java.lang.Object generate(Object[] references) {
/* 003 */ return new SpecificSafeProjection(references);
/* 004 */ }
/* 005 */
/* 006 */ class SpecificSafeProjection extends
org.apache.spark.sql.catalyst.expressions.codegen.BaseProjection {
/* 007 */
/* 008 */ private Object[] references;
/* 009 */ private MutableRow mutableRow;
/* 010 */
/* 011 */
/* 012 */
/* 013 */ public SpecificSafeProjection(Object[] references) {
/* 014 */ this.references = references;
/* 015 */ mutableRow = (MutableRow) references[references.length - 1];
/* 016 */
/* 017 */ }
/* 018 */
/* 019 */ public java.lang.Object apply(java.lang.Object _i) {
/* 020 */ InternalRow i = (InternalRow) _i;
/* 021 */ boolean isNull3 = i.isNullAt(0);
/* 022 */ InternalRow value3 = isNull3 ? null : (i.getStruct(0, 100));
/* 023 */ boolean isNull1 = false;
/* 024 */ org.apache.spark.sql.S100 value1 = null;
/* 025 */ if (!false && isNull3) {
/* 026 */ final org.apache.spark.sql.S100 value4 = null;
/* 027 */ isNull1 = true;
/* 028 */ value1 = value4;
/* 029 */ } else {
/* 030 */ // newInstance(class org.apache.spark.sql.S100)
/* 031 */ Object obj = ((Expression) references[0]).eval(i);
/* 032 */ org.apache.spark.sql.S100 value5 =
(org.apache.spark.sql.S100) obj;
/* 033 */ isNull1 = false;
/* 034 */ value1 = value5;
/* 035 */ }
/* 036 */ boolean isNull8 = i.isNullAt(1);
/* 037 */ InternalRow value8 = isNull8 ? null : (i.getStruct(1, 100));
/* 038 */ boolean isNull6 = false;
/* 039 */ org.apache.spark.sql.S100 value6 = null;
/* 040 */ if (!false && isNull8) {
/* 041 */ final org.apache.spark.sql.S100 value9 = null;
/* 042 */ isNull6 = true;
/* 043 */ value6 = value9;
/* 044 */ } else {
/* 045 */ // newInstance(class org.apache.spark.sql.S100)
/* 046 */ Object obj1 = ((Expression) references[1]).eval(i);
/* 047 */ org.apache.spark.sql.S100 value10 =
(org.apache.spark.sql.S100) obj1;
/* 048 */ isNull6 = false;
/* 049 */ value6 = value10;
/* 050 */ }
/* 051 */ boolean isNull13 = i.isNullAt(2);
/* 052 */ InternalRow value13 = isNull13 ? null : (i.getStruct(2, 100));
/* 053 */ boolean isNull11 = false;
/* 054 */ org.apache.spark.sql.S100 value11 = null;
/* 055 */ if (!false && isNull13) {
/* 056 */ final org.apache.spark.sql.S100 value14 = null;
/* 057 */ isNull11 = true;
/* 058 */ value11 = value14;
/* 059 */ } else {
/* 060 */ // newInstance(class org.apache.spark.sql.S100)
/* 061 */ Object obj2 = ((Expression) references[2]).eval(i);
/* 062 */ org.apache.spark.sql.S100 value15 =
(org.apache.spark.sql.S100) obj2;
/* 063 */ isNull11 = false;
/* 064 */ value11 = value15;
/* 065 */ }
/* 066 */ boolean isNull18 = i.isNullAt(3);
/* 067 */ InternalRow value18 = isNull18 ? null : (i.getStruct(3, 100));
/* 068 */ boolean isNull16 = false;
/* 069 */ org.apache.spark.sql.S100 value16 = null;
/* 070 */ if (!false && isNull18) {
/* 071 */ final org.apache.spark.sql.S100 value19 = null;
/* 072 */ isNull16 = true;
/* 073 */ value16 = value19;
/* 074 */ } else {
/* 075 */ // newInstance(class org.apache.spark.sql.S100)
/* 076 */ Object obj3 = ((Expression) references[3]).eval(i);
/* 077 */ org.apache.spark.sql.S100 value20 =
(org.apache.spark.sql.S100) obj3;
/* 078 */ isNull16 = false;
/* 079 */ value16 = value20;
/* 080 */ }
/* 081 */ boolean isNull23 = i.isNullAt(4);
/* 082 */ InternalRow value23 = isNull23 ? null : (i.getStruct(4, 100));
/* 083 */ boolean isNull21 = false;
/* 084 */ org.apache.spark.sql.S100 value21 = null;
/* 085 */ if (!false && isNull23) {
/* 086 */ final org.apache.spark.sql.S100 value24 = null;
/* 087 */ isNull21 = true;
/* 088 */ value21 = value24;
/* 089 */ } else {
/* 090 */ boolean isNull28 = i.isNullAt(4);
/* 091 */ InternalRow value28 = isNull28 ? null : (i.getStruct(4,
100));
/* 092 */ boolean isNull27 = isNull28;
/* 093 */ UTF8String value27 = null;
/* 094 */
/* 095 */ if (!isNull28) {
/* 096 */
/* 097 */ if (value28.isNullAt(0)) {
/* 098 */ isNull27 = true;
/* 099 */ } else {
/* 100 */ value27 = value28.getUTF8String(0);
/* 101 */ }
/* 102 */
/* 103 */ }
/* 104 */
/* 105 */ boolean isNull26 = isNull27;
/* 106 */ final java.lang.String value26 = isNull26 ? null :
(java.lang.String) value27.toString();
/* 107 */ isNull26 = value26 == null;
/* 108 */ boolean isNull31 = i.isNullAt(4);
/* 109 */ InternalRow value31 = isNull31 ? null : (i.getStruct(4,
100));
/* 110 */ boolean isNull30 = isNull31;
/* 111 */ UTF8String value30 = null;
/* 112 */
/* 113 */ if (!isNull31) {
/* 114 */
/* 115 */ if (value31.isNullAt(1)) {
/* 116 */ isNull30 = true;
/* 117 */ } else {
/* 118 */ value30 = value31.getUTF8String(1);
/* 119 */ }
/* 120 */
/* 121 */ }
/* 122 */
/* 123 */ boolean isNull29 = isNull30;
/* 124 */ final java.lang.String value29 = isNull29 ? null :
(java.lang.String) value30.toString();
/* 125 */ isNull29 = value29 == null;
/* 126 */ boolean isNull34 = i.isNullAt(4);
/* 127 */ InternalRow value34 = isNull34 ? null : (i.getStruct(4,
100));
/* 128 */ boolean isNull33 = isNull34;
/* 129 */ UTF8String value33 = null;
/* 130 */
/* 131 */ if (!isNull34) {
/* 132 */
/* 133 */ if (value34.isNullAt(2)) {
/* 134 */ isNull33 = true;
/* 135 */ } else {
/* 136 */ value33 = value34.getUTF8String(2);
/* 137 */ }
/* 138 */
/* 139 */ }
/* 140 */
/* 141 */ boolean isNull32 = isNull33;
/* 142 */ final java.lang.String value32 = isNull32 ? null :
(java.lang.String) value33.toString();
/* 143 */ isNull32 = value32 == null;
/* 144 */ boolean isNull37 = i.isNullAt(4);
/* 145 */ InternalRow value37 = isNull37 ? null : (i.getStruct(4,
100));
/* 146 */ boolean isNull36 = isNull37;
/* 147 */ UTF8String value36 = null;
/* 148 */
/* 149 */ if (!isNull37) {
/* 150 */
/* 151 */ if (value37.isNullAt(3)) {
/* 152 */ isNull36 = true;
/* 153 */ } else {
/* 154 */ value36 = value37.getUTF8String(3);
/* 155 */ }
/* 156 */
/* 157 */ }
...
/* 1869 */ boolean isNull320 = isNull321;
/* 1870 */ final java.lang.String value320 = isNull320 ? null :
(java.lang.String) value321.toString();
/* 1871 */ isNull320 = value320 == null;
/* 1872 */ boolean isNull325 = i.isNullAt(4);
/* 1873 */ InternalRow value325 = isNull325 ? null : (i.getStruct(4,
100));
/* 1874 */ boolean isNull324 = isNull325;
/* 1875 */ UTF8String value324 = null;
/* 1876 */
/* 1877 */ if (!isNull325) {
/* 1878 */
/* 1879 */ if (value325.isNullAt(99)) {
/* 1880 */ isNull324 = true;
/* 1881 */ } else {
/* 1882 */ value324 = value325.getUTF8String(99);
/* 1883 */ }
/* 1884 */
/* 1885 */ }
/* 1886 */
/* 1887 */ boolean isNull323 = isNull324;
/* 1888 */ final java.lang.String value323 = isNull323 ? null :
(java.lang.String) value324.toString();
/* 1889 */ isNull323 = value323 == null;
/* 1890 */
/* 1891 */
/* 1892 */ final org.apache.spark.sql.S100 value25 = false ? null :
new org.apache.spark.sql.S100(value26, value29, value32, value35, value38,
value41, value44, value47, value50, value53, value56, value59, value62,
value65, value68, value71, value74, value77, value80, value83, value86,
value89, value92, value95, value98, value101, value104, value107, value110,
value113, value116, value119, value122, value125, value128, value131, value134,
value137, value140, value143, value146, value149, value152, value155, value158,
value161, value164, value167, value170, value173, value176, value179, value182,
value185, value188, value191, value194, value197, value200, value203, value206,
value209, value212, value215, value218, value221, value224, value227, value230,
value233, value236, value239, value242, value245, value248, value251, value254,
value257, value260, value263, value266, value269, value272, value275, value278,
value281, value284, value287, value290, value293, value296, value
299, value302, value305, value308, value311, value314, value317, value320,
value323);
/* 1893 */ isNull21 = false;
/* 1894 */ value21 = value25;
/* 1895 */ }
/* 1896 */
/* 1897 */
/* 1898 */ final org.apache.spark.sql.S100_5 value = false ? null : new
org.apache.spark.sql.S100_5(value1, value6, value11, value16, value21);
/* 1899 */ if (false) {
/* 1900 */ mutableRow.setNullAt(0);
/* 1901 */ } else {
/* 1902 */
/* 1903 */ mutableRow.update(0, value);
/* 1904 */ }
/* 1905 */
/* 1906 */ return mutableRow;
/* 1907 */ }
/* 1908 */ }
/* 1909 */
````
## How was this patch tested?
Added new tests
You can merge this pull request into a Git repository by running:
$ git pull https://github.com/kiszk/spark SPARK-15285
Alternatively you can review and apply these changes as the patch at:
https://github.com/apache/spark/pull/13243.patch
To close this pull request, make a commit to your master/trunk branch
with (at least) the following in the commit message:
This closes #13243
----
commit 8c72486b753f94fc8346524fe6cb074b94ae555c
Author: Kazuaki Ishizaki <[email protected]>
Date: 2016-05-21T17:56:26Z
Go to codegen fallback if the total number of NewInstances whose code has
been generated exceed the pre-defined threshold
commit e32a2371b34dc1de8b226b1d58ca76a12a5e580e
Author: Kazuaki Ishizaki <[email protected]>
Date: 2016-05-21T17:59:04Z
revert blank line
----
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at [email protected] or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
