jerqi edited a comment on pull request #33510:
URL: https://github.com/apache/spark/pull/33510#issuecomment-967239979
q97 with shuffleHashJoinConf failed because codegen failed. let me take a
look.
`File 'generated.java', Line 302, Column 1: Thrown exception of type
"java.io.IOException" is neither caught by a "try...catch" block nor declared
in the "throws" clause of the declaring function`
The Complete Codegen Code is as below:
/* 001 */ public Object generate(Object[] references) {
/* 002 */ return new GeneratedIteratorForCodegenStage9(references);
/* 003 */ }
/* 004 */
/* 005 */ // codegenStageId=9
/* 006 */ final class GeneratedIteratorForCodegenStage9 extends
org.apache.spark.sql.execution.BufferedRowIterator {
/* 007 */ private Object[] references;
/* 008 */ private scala.collection.Iterator[] inputs;
/* 009 */ private boolean agg_initAgg_0;
/* 010 */ private boolean agg_bufIsNull_0;
/* 011 */ private long agg_bufValue_0;
/* 012 */ private boolean agg_bufIsNull_1;
/* 013 */ private long agg_bufValue_1;
/* 014 */ private boolean agg_bufIsNull_2;
/* 015 */ private long agg_bufValue_2;
/* 016 */ private boolean agg_initAgg_1;
/* 017 */ private boolean agg_bufIsNull_3;
/* 018 */ private long agg_bufValue_3;
/* 019 */ private boolean agg_bufIsNull_4;
/* 020 */ private long agg_bufValue_4;
/* 021 */ private boolean agg_bufIsNull_5;
/* 022 */ private long agg_bufValue_5;
/* 023 */ private org.apache.spark.sql.execution.joins.HashedRelation
shj_relation_0;
/* 024 */ private boolean shj_keyIsUnique_0;
/* 025 */ private scala.collection.Iterator shj_streamedInput_0;
/* 026 */ private scala.collection.Iterator shj_buildInput_0;
/* 027 */ private InternalRow shj_streamedRow_0;
/* 028 */ private InternalRow shj_buildRow_0;
/* 029 */ private boolean agg_agg_isNull_19_0;
/* 030 */ private boolean agg_agg_isNull_30_0;
/* 031 */ private boolean agg_agg_isNull_41_0;
/* 032 */ private org.apache.spark.util.collection.BitSet
agg_matchedKeySet_0;
/* 033 */ private org.apache.spark.util.collection.OpenHashSet
agg_matchedRowSet_0;
/* 034 */ private int agg_prevKeyIndex_0;
/* 035 */ private int agg_valueIndex_0;
/* 036 */ private boolean agg_agg_isNull_55_0;
/* 037 */ private boolean agg_agg_isNull_57_0;
/* 038 */ private boolean agg_agg_isNull_62_0;
/* 039 */ private boolean agg_agg_isNull_64_0;
/* 040 */ private boolean agg_agg_isNull_69_0;
/* 041 */ private boolean agg_agg_isNull_71_0;
/* 042 */ private
org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[]
shj_mutableStateArray_0 = new
org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[7];
/* 043 */
/* 044 */ public GeneratedIteratorForCodegenStage9(Object[] references) {
/* 045 */ this.references = references;
/* 046 */ }
/* 047 */
/* 048 */ public void init(int index, scala.collection.Iterator[] inputs) {
/* 049 */ partitionIndex = index;
/* 050 */ this.inputs = inputs;
/* 051 */ wholestagecodegen_init_0_0();
/* 052 */ wholestagecodegen_init_0_1();
/* 053 */
/* 054 */ }
/* 055 */
/* 056 */ private void agg_doAggregate_sum_0(boolean agg_exprIsNull_1_0,
int agg_expr_1_0, boolean agg_value_19) throws java.io.IOException {
/* 057 */ agg_agg_isNull_19_0 = true;
/* 058 */ long agg_value_21 = -1L;
/* 059 */ do {
/* 060 */ if (!agg_bufIsNull_3) {
/* 061 */ agg_agg_isNull_19_0 = false;
/* 062 */ agg_value_21 = agg_bufValue_3;
/* 063 */ continue;
/* 064 */ }
/* 065 */
/* 066 */ if (!false) {
/* 067 */ agg_agg_isNull_19_0 = false;
/* 068 */ agg_value_21 = 0L;
/* 069 */ continue;
/* 070 */ }
/* 071 */
/* 072 */ } while (false);
/* 073 */ boolean agg_value_26 = false;
/* 074 */
/* 075 */ if (agg_value_19) {
/* 076 */ agg_value_26 = agg_exprIsNull_1_0;
/* 077 */ }
/* 078 */ boolean agg_isNull_23 = false;
/* 079 */ int agg_value_25 = -1;
/* 080 */ if (!false && agg_value_26) {
/* 081 */ agg_isNull_23 = false;
/* 082 */ agg_value_25 = 1;
/* 083 */ } else {
/* 084 */ agg_isNull_23 = false;
/* 085 */ agg_value_25 = 0;
/* 086 */ }
/* 087 */ boolean agg_isNull_22 = agg_isNull_23;
/* 088 */ long agg_value_24 = -1L;
/* 089 */ if (!agg_isNull_23) {
/* 090 */ agg_value_24 = (long) agg_value_25;
/* 091 */ }
/* 092 */ long agg_value_20 = -1L;
/* 093 */
/* 094 */ agg_value_20 = agg_value_21 + agg_value_24;
/* 095 */
/* 096 */ agg_bufIsNull_3 = false;
/* 097 */ agg_bufValue_3 = agg_value_20;
/* 098 */ }
/* 099 */
/* 100 */ private void agg_doAggregate_sum_3(long agg_expr_0_1, boolean
agg_exprIsNull_0_1) throws java.io.IOException {
/* 101 */ agg_agg_isNull_55_0 = true;
/* 102 */ long agg_value_57 = -1L;
/* 103 */ do {
/* 104 */ boolean agg_isNull_56 = true;
/* 105 */ long agg_value_58 = -1L;
/* 106 */ agg_agg_isNull_57_0 = true;
/* 107 */ long agg_value_59 = -1L;
/* 108 */ do {
/* 109 */ if (!agg_bufIsNull_0) {
/* 110 */ agg_agg_isNull_57_0 = false;
/* 111 */ agg_value_59 = agg_bufValue_0;
/* 112 */ continue;
/* 113 */ }
/* 114 */
/* 115 */ if (!false) {
/* 116 */ agg_agg_isNull_57_0 = false;
/* 117 */ agg_value_59 = 0L;
/* 118 */ continue;
/* 119 */ }
/* 120 */
/* 121 */ } while (false);
/* 122 */
/* 123 */ if (!agg_exprIsNull_0_1) {
/* 124 */ agg_isNull_56 = false; // resultCode could change
nullability.
/* 125 */
/* 126 */ agg_value_58 = agg_value_59 + agg_expr_0_1;
/* 127 */
/* 128 */ }
/* 129 */ if (!agg_isNull_56) {
/* 130 */ agg_agg_isNull_55_0 = false;
/* 131 */ agg_value_57 = agg_value_58;
/* 132 */ continue;
/* 133 */ }
/* 134 */
/* 135 */ if (!agg_bufIsNull_0) {
/* 136 */ agg_agg_isNull_55_0 = false;
/* 137 */ agg_value_57 = agg_bufValue_0;
/* 138 */ continue;
/* 139 */ }
/* 140 */
/* 141 */ } while (false);
/* 142 */
/* 143 */ agg_bufIsNull_0 = agg_agg_isNull_55_0;
/* 144 */ agg_bufValue_0 = agg_value_57;
/* 145 */ }
/* 146 */
/* 147 */ private void agg_doAggregateWithoutKey_0() throws
java.io.IOException {
/* 148 */ // initialize aggregation buffer
/* 149 */ agg_bufIsNull_0 = true;
/* 150 */ agg_bufValue_0 = -1L;
/* 151 */ agg_bufIsNull_1 = true;
/* 152 */ agg_bufValue_1 = -1L;
/* 153 */ agg_bufIsNull_2 = true;
/* 154 */ agg_bufValue_2 = -1L;
/* 155 */
/* 156 */ while (!agg_initAgg_1) {
/* 157 */ agg_initAgg_1 = true;
/* 158 */ long agg_beforeAgg_0 = System.nanoTime();
/* 159 */ agg_doAggregateWithoutKey_1();
/* 160 */ ((org.apache.spark.sql.execution.metric.SQLMetric)
references[3] /* aggTime */).add((System.nanoTime() - agg_beforeAgg_0) /
1000000);
/* 161 */
/* 162 */ // output the result
/* 163 */
/* 164 */ ((org.apache.spark.sql.execution.metric.SQLMetric)
references[2] /* numOutputRows */).add(1);
/* 165 */ agg_doConsume_1(agg_bufValue_3, agg_bufIsNull_3,
agg_bufValue_4, agg_bufIsNull_4, agg_bufValue_5, agg_bufIsNull_5);
/* 166 */ }
/* 167 */
/* 168 */ }
/* 169 */
/* 170 */ private void agg_doAggregate_sum_2(boolean agg_value_19, boolean
agg_value_16) throws java.io.IOException {
/* 171 */ agg_agg_isNull_41_0 = true;
/* 172 */ long agg_value_43 = -1L;
/* 173 */ do {
/* 174 */ if (!agg_bufIsNull_5) {
/* 175 */ agg_agg_isNull_41_0 = false;
/* 176 */ agg_value_43 = agg_bufValue_5;
/* 177 */ continue;
/* 178 */ }
/* 179 */
/* 180 */ if (!false) {
/* 181 */ agg_agg_isNull_41_0 = false;
/* 182 */ agg_value_43 = 0L;
/* 183 */ continue;
/* 184 */ }
/* 185 */
/* 186 */ } while (false);
/* 187 */ boolean agg_value_48 = false;
/* 188 */
/* 189 */ if (agg_value_19) {
/* 190 */ agg_value_48 = agg_value_16;
/* 191 */ }
/* 192 */ boolean agg_isNull_45 = false;
/* 193 */ int agg_value_47 = -1;
/* 194 */ if (!false && agg_value_48) {
/* 195 */ agg_isNull_45 = false;
/* 196 */ agg_value_47 = 1;
/* 197 */ } else {
/* 198 */ agg_isNull_45 = false;
/* 199 */ agg_value_47 = 0;
/* 200 */ }
/* 201 */ boolean agg_isNull_44 = agg_isNull_45;
/* 202 */ long agg_value_46 = -1L;
/* 203 */ if (!agg_isNull_45) {
/* 204 */ agg_value_46 = (long) agg_value_47;
/* 205 */ }
/* 206 */ long agg_value_42 = -1L;
/* 207 */
/* 208 */ agg_value_42 = agg_value_43 + agg_value_46;
/* 209 */
/* 210 */ agg_bufIsNull_5 = false;
/* 211 */ agg_bufValue_5 = agg_value_42;
/* 212 */ }
/* 213 */
/* 214 */ private void agg_doConsume_1(long agg_expr_0_1, boolean
agg_exprIsNull_0_1, long agg_expr_1_1, boolean agg_exprIsNull_1_1, long
agg_expr_2_0, boolean agg_exprIsNull_2_0) throws java.io.IOException {
/* 215 */ // do aggregate
/* 216 */ // common sub-expressions
/* 217 */
/* 218 */ // evaluate aggregate functions and update aggregation buffers
/* 219 */ agg_doAggregate_sum_3(agg_expr_0_1, agg_exprIsNull_0_1);
/* 220 */ agg_doAggregate_sum_4(agg_expr_1_1, agg_exprIsNull_1_1);
/* 221 */ agg_doAggregate_sum_5(agg_exprIsNull_2_0, agg_expr_2_0);
/* 222 */
/* 223 */ }
/* 224 */
/* 225 */ private void agg_doAggregate_sum_5(boolean agg_exprIsNull_2_0,
long agg_expr_2_0) throws java.io.IOException {
/* 226 */ agg_agg_isNull_69_0 = true;
/* 227 */ long agg_value_71 = -1L;
/* 228 */ do {
/* 229 */ boolean agg_isNull_70 = true;
/* 230 */ long agg_value_72 = -1L;
/* 231 */ agg_agg_isNull_71_0 = true;
/* 232 */ long agg_value_73 = -1L;
/* 233 */ do {
/* 234 */ if (!agg_bufIsNull_2) {
/* 235 */ agg_agg_isNull_71_0 = false;
/* 236 */ agg_value_73 = agg_bufValue_2;
/* 237 */ continue;
/* 238 */ }
/* 239 */
/* 240 */ if (!false) {
/* 241 */ agg_agg_isNull_71_0 = false;
/* 242 */ agg_value_73 = 0L;
/* 243 */ continue;
/* 244 */ }
/* 245 */
/* 246 */ } while (false);
/* 247 */
/* 248 */ if (!agg_exprIsNull_2_0) {
/* 249 */ agg_isNull_70 = false; // resultCode could change
nullability.
/* 250 */
/* 251 */ agg_value_72 = agg_value_73 + agg_expr_2_0;
/* 252 */
/* 253 */ }
/* 254 */ if (!agg_isNull_70) {
/* 255 */ agg_agg_isNull_69_0 = false;
/* 256 */ agg_value_71 = agg_value_72;
/* 257 */ continue;
/* 258 */ }
/* 259 */
/* 260 */ if (!agg_bufIsNull_2) {
/* 261 */ agg_agg_isNull_69_0 = false;
/* 262 */ agg_value_71 = agg_bufValue_2;
/* 263 */ continue;
/* 264 */ }
/* 265 */
/* 266 */ } while (false);
/* 267 */
/* 268 */ agg_bufIsNull_2 = agg_agg_isNull_69_0;
/* 269 */ agg_bufValue_2 = agg_value_71;
/* 270 */ }
/* 271 */
/* 272 */ private void wholestagecodegen_init_0_1() {
/* 273 */ shj_mutableStateArray_0[5] = new
org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(3, 0);
/* 274 */ shj_mutableStateArray_0[6] = new
org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(3, 0);
/* 275 */
/* 276 */ }
/* 277 */
/* 278 */ private void shj_consumeFullOuterJoinRow_0() {
/* 279 */ ((org.apache.spark.sql.execution.metric.SQLMetric)
references[1] /* numOutputRows */).add(1);
/* 280 */
/* 281 */ // common sub-expressions
/* 282 */
/* 283 */ boolean shj_isNull_14 = true;
/* 284 */ int shj_value_14 = -1;
/* 285 */ if (shj_streamedRow_0 != null) {
/* 286 */ boolean shj_isNull_13 = shj_streamedRow_0.isNullAt(0);
/* 287 */ int shj_value_13 = shj_isNull_13 ?
/* 288 */ -1 : (shj_streamedRow_0.getInt(0));
/* 289 */ shj_isNull_14 = shj_isNull_13;
/* 290 */ shj_value_14 = shj_value_13;
/* 291 */ }
/* 292 */ boolean shj_isNull_18 = true;
/* 293 */ int shj_value_18 = -1;
/* 294 */ if (shj_buildRow_0 != null) {
/* 295 */ boolean shj_isNull_17 = shj_buildRow_0.isNullAt(0);
/* 296 */ int shj_value_17 = shj_isNull_17 ?
/* 297 */ -1 : (shj_buildRow_0.getInt(0));
/* 298 */ shj_isNull_18 = shj_isNull_17;
/* 299 */ shj_value_18 = shj_value_17;
/* 300 */ }
/* 301 */
/* 302 */ agg_doConsume_0(shj_value_14, shj_isNull_14, shj_value_18,
shj_isNull_18);
/* 303 */
/* 304 */ }
/* 305 */
/* 306 */ private void agg_doAggregate_sum_1(int agg_expr_0_0, boolean
agg_exprIsNull_0_0, boolean agg_value_16) throws java.io.IOException {
/* 307 */ agg_agg_isNull_30_0 = true;
/* 308 */ long agg_value_32 = -1L;
/* 309 */ do {
/* 310 */ if (!agg_bufIsNull_4) {
/* 311 */ agg_agg_isNull_30_0 = false;
/* 312 */ agg_value_32 = agg_bufValue_4;
/* 313 */ continue;
/* 314 */ }
/* 315 */
/* 316 */ if (!false) {
/* 317 */ agg_agg_isNull_30_0 = false;
/* 318 */ agg_value_32 = 0L;
/* 319 */ continue;
/* 320 */ }
/* 321 */
/* 322 */ } while (false);
/* 323 */ boolean agg_value_37 = false;
/* 324 */
/* 325 */ if (agg_exprIsNull_0_0) {
/* 326 */ agg_value_37 = agg_value_16;
/* 327 */ }
/* 328 */ boolean agg_isNull_34 = false;
/* 329 */ int agg_value_36 = -1;
/* 330 */ if (!false && agg_value_37) {
/* 331 */ agg_isNull_34 = false;
/* 332 */ agg_value_36 = 1;
/* 333 */ } else {
/* 334 */ agg_isNull_34 = false;
/* 335 */ agg_value_36 = 0;
/* 336 */ }
/* 337 */ boolean agg_isNull_33 = agg_isNull_34;
/* 338 */ long agg_value_35 = -1L;
/* 339 */ if (!agg_isNull_34) {
/* 340 */ agg_value_35 = (long) agg_value_36;
/* 341 */ }
/* 342 */ long agg_value_31 = -1L;
/* 343 */
/* 344 */ agg_value_31 = agg_value_32 + agg_value_35;
/* 345 */
/* 346 */ agg_bufIsNull_4 = false;
/* 347 */ agg_bufValue_4 = agg_value_31;
/* 348 */ }
/* 349 */
/* 350 */ private void agg_doConsume_0(int agg_expr_0_0, boolean
agg_exprIsNull_0_0, int agg_expr_1_0, boolean agg_exprIsNull_1_0) throws
java.io.IOException {
/* 351 */ // do aggregate
/* 352 */ // common sub-expressions
/* 353 */
/* 354 */ boolean agg_value_16 = !agg_exprIsNull_1_0;
/* 355 */
/* 356 */ boolean agg_value_19 = !agg_exprIsNull_0_0;
/* 357 */
/* 358 */ // evaluate aggregate functions and update aggregation buffers
/* 359 */ agg_doAggregate_sum_0(agg_exprIsNull_1_0, agg_expr_1_0,
agg_value_19);
/* 360 */ agg_doAggregate_sum_1(agg_expr_0_0, agg_exprIsNull_0_0,
agg_value_16);
/* 361 */ agg_doAggregate_sum_2(agg_value_19, agg_value_16);
/* 362 */
/* 363 */ }
/* 364 */
/* 365 */ private void agg_doAggregate_sum_4(long agg_expr_1_1, boolean
agg_exprIsNull_1_1) throws java.io.IOException {
/* 366 */ agg_agg_isNull_62_0 = true;
/* 367 */ long agg_value_64 = -1L;
/* 368 */ do {
/* 369 */ boolean agg_isNull_63 = true;
/* 370 */ long agg_value_65 = -1L;
/* 371 */ agg_agg_isNull_64_0 = true;
/* 372 */ long agg_value_66 = -1L;
/* 373 */ do {
/* 374 */ if (!agg_bufIsNull_1) {
/* 375 */ agg_agg_isNull_64_0 = false;
/* 376 */ agg_value_66 = agg_bufValue_1;
/* 377 */ continue;
/* 378 */ }
/* 379 */
/* 380 */ if (!false) {
/* 381 */ agg_agg_isNull_64_0 = false;
/* 382 */ agg_value_66 = 0L;
/* 383 */ continue;
/* 384 */ }
/* 385 */
/* 386 */ } while (false);
/* 387 */
/* 388 */ if (!agg_exprIsNull_1_1) {
/* 389 */ agg_isNull_63 = false; // resultCode could change
nullability.
/* 390 */
/* 391 */ agg_value_65 = agg_value_66 + agg_expr_1_1;
/* 392 */
/* 393 */ }
/* 394 */ if (!agg_isNull_63) {
/* 395 */ agg_agg_isNull_62_0 = false;
/* 396 */ agg_value_64 = agg_value_65;
/* 397 */ continue;
/* 398 */ }
/* 399 */
/* 400 */ if (!agg_bufIsNull_1) {
/* 401 */ agg_agg_isNull_62_0 = false;
/* 402 */ agg_value_64 = agg_bufValue_1;
/* 403 */ continue;
/* 404 */ }
/* 405 */
/* 406 */ } while (false);
/* 407 */
/* 408 */ agg_bufIsNull_1 = agg_agg_isNull_62_0;
/* 409 */ agg_bufValue_1 = agg_value_64;
/* 410 */ }
/* 411 */
/* 412 */ private void agg_doAggregateWithoutKey_1() throws
java.io.IOException {
/* 413 */ // initialize aggregation buffer
/* 414 */ agg_bufIsNull_3 = true;
/* 415 */ agg_bufValue_3 = -1L;
/* 416 */ agg_bufIsNull_4 = true;
/* 417 */ agg_bufValue_4 = -1L;
/* 418 */ agg_bufIsNull_5 = true;
/* 419 */ agg_bufValue_5 = -1L;
/* 420 */
/* 421 */ if (shj_keyIsUnique_0) {
/* 422 */ while (shj_streamedInput_0.hasNext()) {
/* 423 */ shj_streamedRow_0 = (InternalRow)
shj_streamedInput_0.next();
/* 424 */
/* 425 */ // generate join key for stream side
/* 426 */
/* 427 */ boolean shj_isNull_0 = shj_streamedRow_0.isNullAt(0);
/* 428 */ int shj_value_0 = shj_isNull_0 ?
/* 429 */ -1 : (shj_streamedRow_0.getInt(0));
/* 430 */ boolean shj_isNull_1 = shj_streamedRow_0.isNullAt(1);
/* 431 */ int shj_value_1 = shj_isNull_1 ?
/* 432 */ -1 : (shj_streamedRow_0.getInt(1));
/* 433 */
/* 434 */ shj_mutableStateArray_0[0].reset();
/* 435 */
/* 436 */ shj_mutableStateArray_0[0].zeroOutNullBytes();
/* 437 */
/* 438 */ boolean shj_isNull_2 = true;
/* 439 */ long shj_value_2 = -1L;
/* 440 */ boolean shj_isNull_3 = true;
/* 441 */ long shj_value_3 = -1L;
/* 442 */ boolean shj_isNull_4 = shj_isNull_0;
/* 443 */ long shj_value_4 = -1L;
/* 444 */ if (!shj_isNull_0) {
/* 445 */ shj_value_4 = (long) shj_value_0;
/* 446 */ }
/* 447 */ if (!shj_isNull_4) {
/* 448 */ shj_isNull_3 = false; // resultCode could change
nullability.
/* 449 */ shj_value_3 = shj_value_4 << 32;
/* 450 */
/* 451 */ }
/* 452 */ if (!shj_isNull_3) {
/* 453 */ boolean shj_isNull_7 = true;
/* 454 */ long shj_value_7 = -1L;
/* 455 */ boolean shj_isNull_8 = shj_isNull_1;
/* 456 */ long shj_value_8 = -1L;
/* 457 */ if (!shj_isNull_1) {
/* 458 */ shj_value_8 = (long) shj_value_1;
/* 459 */ }
/* 460 */ if (!shj_isNull_8) {
/* 461 */ shj_isNull_7 = false; // resultCode could change
nullability.
/* 462 */
/* 463 */ shj_value_7 = shj_value_8 & 4294967295L;
/* 464 */
/* 465 */ }
/* 466 */ if (!shj_isNull_7) {
/* 467 */ shj_isNull_2 = false; // resultCode could change
nullability.
/* 468 */
/* 469 */ shj_value_2 = shj_value_3 | shj_value_7;
/* 470 */
/* 471 */ }
/* 472 */
/* 473 */ }
/* 474 */ if (shj_isNull_2) {
/* 475 */ shj_mutableStateArray_0[0].setNullAt(0);
/* 476 */ } else {
/* 477 */ shj_mutableStateArray_0[0].write(0, shj_value_2);
/* 478 */ }
/* 479 */
/* 480 */ // find matches from HashedRelation
/* 481 */ boolean agg_foundMatch_0 = false;
/* 482 */ shj_buildRow_0 = null;
/* 483 */ org.apache.spark.sql.execution.joins.ValueRowWithKeyIndex
agg_rowWithIndex_0 = (shj_mutableStateArray_0[0].getRow()).anyNull() ? null:
/* 484 */
shj_relation_0.getValueWithKeyIndex((shj_mutableStateArray_0[0].getRow()));
/* 485 */
/* 486 */ if (agg_rowWithIndex_0 != null) {
/* 487 */ shj_buildRow_0 = agg_rowWithIndex_0.getValue();
/* 488 */ // check join condition
/* 489 */ {
/* 490 */ // set key index in matched keys set
/* 491 */
agg_matchedKeySet_0.set(agg_rowWithIndex_0.getKeyIndex());
/* 492 */ agg_foundMatch_0 = true;
/* 493 */ }
/* 494 */
/* 495 */ if (!agg_foundMatch_0) {
/* 496 */ shj_buildRow_0 = null;
/* 497 */ }
/* 498 */ }
/* 499 */
/* 500 */ shj_consumeFullOuterJoinRow_0();
/* 501 */ if (shouldStop()) return;
/* 502 */ }
/* 503 */
/* 504 */ shj_streamedRow_0 = null;
/* 505 */
/* 506 */ // find non-matched rows from HashedRelation
/* 507 */ while (shj_buildInput_0.hasNext()) {
/* 508 */ org.apache.spark.sql.execution.joins.ValueRowWithKeyIndex
agg_rowWithIndex_0 =
(org.apache.spark.sql.execution.joins.ValueRowWithKeyIndex)
shj_buildInput_0.next();
/* 509 */
/* 510 */ // check if key index is not in matched keys set
/* 511 */ if
(!agg_matchedKeySet_0.get(agg_rowWithIndex_0.getKeyIndex())) {
/* 512 */ shj_buildRow_0 = agg_rowWithIndex_0.getValue();
/* 513 */ shj_consumeFullOuterJoinRow_0();
/* 514 */ }
/* 515 */
/* 516 */ if (shouldStop()) return;
/* 517 */ }
/* 518 */
/* 519 */ } else {
/* 520 */ while (shj_streamedInput_0.hasNext()) {
/* 521 */ shj_streamedRow_0 = (InternalRow)
shj_streamedInput_0.next();
/* 522 */
/* 523 */ // generate join key for stream side
/* 524 */
/* 525 */ boolean shj_isNull_0 = shj_streamedRow_0.isNullAt(0);
/* 526 */ int shj_value_0 = shj_isNull_0 ?
/* 527 */ -1 : (shj_streamedRow_0.getInt(0));
/* 528 */ boolean shj_isNull_1 = shj_streamedRow_0.isNullAt(1);
/* 529 */ int shj_value_1 = shj_isNull_1 ?
/* 530 */ -1 : (shj_streamedRow_0.getInt(1));
/* 531 */
/* 532 */ shj_mutableStateArray_0[0].reset();
/* 533 */
/* 534 */ shj_mutableStateArray_0[0].zeroOutNullBytes();
/* 535 */
/* 536 */ boolean shj_isNull_2 = true;
/* 537 */ long shj_value_2 = -1L;
/* 538 */ boolean shj_isNull_3 = true;
/* 539 */ long shj_value_3 = -1L;
/* 540 */ boolean shj_isNull_4 = shj_isNull_0;
/* 541 */ long shj_value_4 = -1L;
/* 542 */ if (!shj_isNull_0) {
/* 543 */ shj_value_4 = (long) shj_value_0;
/* 544 */ }
/* 545 */ if (!shj_isNull_4) {
/* 546 */ shj_isNull_3 = false; // resultCode could change
nullability.
/* 547 */ shj_value_3 = shj_value_4 << 32;
/* 548 */
/* 549 */ }
/* 550 */ if (!shj_isNull_3) {
/* 551 */ boolean shj_isNull_7 = true;
/* 552 */ long shj_value_7 = -1L;
/* 553 */ boolean shj_isNull_8 = shj_isNull_1;
/* 554 */ long shj_value_8 = -1L;
/* 555 */ if (!shj_isNull_1) {
/* 556 */ shj_value_8 = (long) shj_value_1;
/* 557 */ }
/* 558 */ if (!shj_isNull_8) {
/* 559 */ shj_isNull_7 = false; // resultCode could change
nullability.
/* 560 */
/* 561 */ shj_value_7 = shj_value_8 & 4294967295L;
/* 562 */
/* 563 */ }
/* 564 */ if (!shj_isNull_7) {
/* 565 */ shj_isNull_2 = false; // resultCode could change
nullability.
/* 566 */
/* 567 */ shj_value_2 = shj_value_3 | shj_value_7;
/* 568 */
/* 569 */ }
/* 570 */
/* 571 */ }
/* 572 */ if (shj_isNull_2) {
/* 573 */ shj_mutableStateArray_0[0].setNullAt(0);
/* 574 */ } else {
/* 575 */ shj_mutableStateArray_0[0].write(0, shj_value_2);
/* 576 */ }
/* 577 */
/* 578 */ // find matches from HashedRelation
/* 579 */ boolean agg_foundMatch_1 = false;
/* 580 */ shj_buildRow_0 = null;
/* 581 */ scala.collection.Iterator agg_buildIterator_0 =
(shj_mutableStateArray_0[0].getRow()).anyNull() ? null:
/* 582 */
shj_relation_0.getWithKeyIndex((shj_mutableStateArray_0[0].getRow()));
/* 583 */
/* 584 */ int agg_valueIndex_0 = -1;
/* 585 */ while (agg_buildIterator_0 != null &&
agg_buildIterator_0.hasNext()) {
/* 586 */
org.apache.spark.sql.execution.joins.ValueRowWithKeyIndex agg_rowWithIndex_1 =
(org.apache.spark.sql.execution.joins.ValueRowWithKeyIndex)
agg_buildIterator_0.next();
/* 587 */ int agg_keyIndex_0 = agg_rowWithIndex_1.getKeyIndex();
/* 588 */ shj_buildRow_0 = agg_rowWithIndex_1.getValue();
/* 589 */ agg_valueIndex_0++;
/* 590 */
/* 591 */ // check join condition
/* 592 */ {
/* 593 */ // set row index in matched row set
/* 594 */ agg_matchedRowSet_0.add((((long)agg_keyIndex_0) << 32)
| agg_valueIndex_0);
/* 595 */ agg_foundMatch_1 = true;
/* 596 */ shj_consumeFullOuterJoinRow_0();
/* 597 */ }
/* 598 */ }
/* 599 */
/* 600 */ if (!agg_foundMatch_1) {
/* 601 */ shj_buildRow_0 = null;
/* 602 */ shj_consumeFullOuterJoinRow_0();
/* 603 */ }
/* 604 */
/* 605 */ if (shouldStop()) return;
/* 606 */ }
/* 607 */
/* 608 */ shj_streamedRow_0 = null;
/* 609 */
/* 610 */ // find non-matched rows from HashedRelation
/* 611 */ while (shj_buildInput_0.hasNext()) {
/* 612 */ org.apache.spark.sql.execution.joins.ValueRowWithKeyIndex
agg_rowWithIndex_1 =
(org.apache.spark.sql.execution.joins.ValueRowWithKeyIndex)
shj_buildInput_0.next();
/* 613 */ int agg_keyIndex_0 = agg_rowWithIndex_1.getKeyIndex();
/* 614 */ if (agg_prevKeyIndex_0 == -1 || agg_keyIndex_0 !=
agg_prevKeyIndex_0) {
/* 615 */ agg_valueIndex_0 = 0;
/* 616 */ agg_prevKeyIndex_0 = agg_keyIndex_0;
/* 617 */ } else {
/* 618 */ agg_valueIndex_0 += 1;
/* 619 */ }
/* 620 */
/* 621 */ // check if row index is not in matched row set
/* 622 */ if (!agg_matchedRowSet_0.contains((((long)agg_keyIndex_0)
<< 32) | agg_valueIndex_0)) {
/* 623 */ shj_buildRow_0 = agg_rowWithIndex_1.getValue();
/* 624 */ shj_consumeFullOuterJoinRow_0();
/* 625 */ }
/* 626 */
/* 627 */ if (shouldStop()) return;
/* 628 */ }
/* 629 */
/* 630 */ }
/* 631 */
/* 632 */ }
/* 633 */
/* 634 */ protected void processNext() throws java.io.IOException {
/* 635 */ while (!agg_initAgg_0) {
/* 636 */ agg_initAgg_0 = true;
/* 637 */ long agg_beforeAgg_1 = System.nanoTime();
/* 638 */ agg_doAggregateWithoutKey_0();
/* 639 */ ((org.apache.spark.sql.execution.metric.SQLMetric)
references[5] /* aggTime */).add((System.nanoTime() - agg_beforeAgg_1) /
1000000);
/* 640 */
/* 641 */ // output the result
/* 642 */
/* 643 */ ((org.apache.spark.sql.execution.metric.SQLMetric)
references[4] /* numOutputRows */).add(1);
/* 644 */ shj_mutableStateArray_0[6].reset();
/* 645 */
/* 646 */ shj_mutableStateArray_0[6].zeroOutNullBytes();
/* 647 */
/* 648 */ if (agg_bufIsNull_0) {
/* 649 */ shj_mutableStateArray_0[6].setNullAt(0);
/* 650 */ } else {
/* 651 */ shj_mutableStateArray_0[6].write(0, agg_bufValue_0);
/* 652 */ }
/* 653 */
/* 654 */ if (agg_bufIsNull_1) {
/* 655 */ shj_mutableStateArray_0[6].setNullAt(1);
/* 656 */ } else {
/* 657 */ shj_mutableStateArray_0[6].write(1, agg_bufValue_1);
/* 658 */ }
/* 659 */
/* 660 */ if (agg_bufIsNull_2) {
/* 661 */ shj_mutableStateArray_0[6].setNullAt(2);
/* 662 */ } else {
/* 663 */ shj_mutableStateArray_0[6].write(2, agg_bufValue_2);
/* 664 */ }
/* 665 */ append((shj_mutableStateArray_0[6].getRow()));
/* 666 */ }
/* 667 */ }
/* 668 */
/* 669 */ private void wholestagecodegen_init_0_0() {
/* 670 */ shj_relation_0 =
((org.apache.spark.sql.execution.joins.ShuffledHashJoinExec) references[0] /*
plan */).buildHashedRelation(inputs[1]);
/* 671 */ shj_keyIsUnique_0 = shj_relation_0.keyIsUnique();
/* 672 */ shj_streamedInput_0 = inputs[0];
/* 673 */ shj_buildInput_0 = shj_relation_0.valuesWithKeyIndex();
/* 674 */ shj_mutableStateArray_0[0] = new
org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(1, 0);
/* 675 */ shj_mutableStateArray_0[1] = new
org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(4, 0);
/* 676 */ shj_mutableStateArray_0[2] = new
org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(2, 0);
/* 677 */ shj_mutableStateArray_0[3] = new
org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(2, 0);
/* 678 */ agg_matchedKeySet_0 = new
org.apache.spark.util.collection.BitSet(shj_relation_0.maxNumKeysIndex());
/* 679 */ agg_matchedRowSet_0 = new
org.apache.spark.util.collection.OpenHashSet(scala.reflect.ClassTag$.MODULE$.Long());
/* 680 */ agg_prevKeyIndex_0 = -1;
/* 681 */ agg_valueIndex_0 = -1;
/* 682 */ shj_mutableStateArray_0[4] = new
org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(3, 0);
/* 683 */
/* 684 */ }
/* 685 */
/* 686 */ }
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
