GitHub user heary-cao opened a pull request:
https://github.com/apache/spark/pull/21860
[SPARK-24901][SQL]Merge the codegen of RegularHashMap and fastHashMap to
reduce compiler maxCodesize when VectorizedHashMap is false.
## What changes were proposed in this pull request?
Currently, Generate code of update UnsafeRow in hash aggregation.
FastHashMap and RegularHashMap are two separate codesï¼These two separate
codes need only when VectorizedHashMap is true. but other cases, we can merge
together to reduce compiler maxCodesize. thanks.
case class DistinctAgg(a: Int, b: Float, c: Double, d: Int, e: String)
spark.sparkContext.parallelize(
DistinctAgg(8, 2, 3, 4, "a") ::
DistinctAgg(9, 3, 4, 5, "b")
::Nil).toDF()createOrReplaceTempView("distinctAgg")
val df = sql("select a,b,e, min(d) as mind, min(case when a > 10 then a
else null end) as mincasea, min(a) as mina from distinctAgg group by a, b, e")
println(org.apache.spark.sql.execution.debug.codegenString(df.queryExecution.executedPlan))
df.show()
Generate code like:
Before modified:
Generated code:
/* 001 */ public Object generate(Object[] references) {
/* 002 */ return new GeneratedIteratorForCodegenStage1(references);
/* 003 */ }
/* 004 */
...............
/* 354 */
/* 355 */ if (agg_fastAggBuffer_0 != null) {
/* 356 */ // common sub-expressions
/* 357 */
/* 358 */ // evaluate aggregate function
/* 359 */ agg_agg_isNull_31_0 = true;
/* 360 */ int agg_value_34 = -1;
/* 361 */
/* 362 */ boolean agg_isNull_32 = agg_fastAggBuffer_0.isNullAt(0);
/* 363 */ int agg_value_35 = agg_isNull_32 ?
/* 364 */ -1 : (agg_fastAggBuffer_0.getInt(0));
/* 365 */
/* 366 */ if (!agg_isNull_32 && (agg_agg_isNull_31_0 ||
/* 367 */ agg_value_34 > agg_value_35)) {
/* 368 */ agg_agg_isNull_31_0 = false;
/* 369 */ agg_value_34 = agg_value_35;
/* 370 */ }
/* 371 */
/* 372 */ if (!false && (agg_agg_isNull_31_0 ||
/* 373 */ agg_value_34 > agg_expr_2_0)) {
/* 374 */ agg_agg_isNull_31_0 = false;
/* 375 */ agg_value_34 = agg_expr_2_0;
/* 376 */ }
/* 377 */ agg_agg_isNull_34_0 = true;
/* 378 */ int agg_value_37 = -1;
/* 379 */
/* 380 */ boolean agg_isNull_35 = agg_fastAggBuffer_0.isNullAt(1);
/* 381 */ int agg_value_38 = agg_isNull_35 ?
/* 382 */ -1 : (agg_fastAggBuffer_0.getInt(1));
/* 383 */
/* 384 */ if (!agg_isNull_35 && (agg_agg_isNull_34_0 ||
/* 385 */ agg_value_37 > agg_value_38)) {
/* 386 */ agg_agg_isNull_34_0 = false;
/* 387 */ agg_value_37 = agg_value_38;
/* 388 */ }
/* 389 */
/* 390 */ byte agg_caseWhenResultState_1 = -1;
/* 391 */ do {
/* 392 */ boolean agg_value_40 = false;
/* 393 */ agg_value_40 = agg_expr_0_0 > 10;
/* 394 */ if (!false && agg_value_40) {
/* 395 */ agg_caseWhenResultState_1 = (byte)(false ? 1 : 0);
/* 396 */ agg_agg_value_39_0 = agg_expr_0_0;
/* 397 */ continue;
/* 398 */ }
/* 399 */
/* 400 */ agg_caseWhenResultState_1 = (byte)(true ? 1 : 0);
/* 401 */ agg_agg_value_39_0 = -1;
/* 402 */
/* 403 */ } while (false);
/* 404 */ // TRUE if any condition is met and the result is null, or
no any condition is met.
/* 405 */ final boolean agg_isNull_36 = (agg_caseWhenResultState_1 !=
0);
/* 406 */
/* 407 */ if (!agg_isNull_36 && (agg_agg_isNull_34_0 ||
/* 408 */ agg_value_37 > agg_agg_value_39_0)) {
/* 409 */ agg_agg_isNull_34_0 = false;
/* 410 */ agg_value_37 = agg_agg_value_39_0;
/* 411 */ }
/* 412 */ agg_agg_isNull_42_0 = true;
/* 413 */ int agg_value_45 = -1;
/* 414 */
/* 415 */ boolean agg_isNull_43 = agg_fastAggBuffer_0.isNullAt(2);
/* 416 */ int agg_value_46 = agg_isNull_43 ?
/* 417 */ -1 : (agg_fastAggBuffer_0.getInt(2));
/* 418 */
/* 419 */ if (!agg_isNull_43 && (agg_agg_isNull_42_0 ||
/* 420 */ agg_value_45 > agg_value_46)) {
/* 421 */ agg_agg_isNull_42_0 = false;
/* 422 */ agg_value_45 = agg_value_46;
/* 423 */ }
/* 424 */
/* 425 */ if (!false && (agg_agg_isNull_42_0 ||
/* 426 */ agg_value_45 > agg_expr_0_0)) {
/* 427 */ agg_agg_isNull_42_0 = false;
/* 428 */ agg_value_45 = agg_expr_0_0;
/* 429 */ }
/* 430 */ // update fast row
/* 431 */ agg_fastAggBuffer_0.setInt(0, agg_value_34);
/* 432 */
/* 433 */ if (!agg_agg_isNull_34_0) {
/* 434 */ agg_fastAggBuffer_0.setInt(1, agg_value_37);
/* 435 */ } else {
/* 436 */ agg_fastAggBuffer_0.setNullAt(1);
/* 437 */ }
/* 438 */
/* 439 */ agg_fastAggBuffer_0.setInt(2, agg_value_45);
/* 440 */ } else {
/* 441 */ // common sub-expressions
/* 442 */
/* 443 */ // evaluate aggregate function
/* 444 */ agg_agg_isNull_17_0 = true;
/* 445 */ int agg_value_20 = -1;
/* 446 */
/* 447 */ boolean agg_isNull_18 =
agg_unsafeRowAggBuffer_0.isNullAt(0);
/* 448 */ int agg_value_21 = agg_isNull_18 ?
/* 449 */ -1 : (agg_unsafeRowAggBuffer_0.getInt(0));
/* 450 */
/* 451 */ if (!agg_isNull_18 && (agg_agg_isNull_17_0 ||
/* 452 */ agg_value_20 > agg_value_21)) {
/* 453 */ agg_agg_isNull_17_0 = false;
/* 454 */ agg_value_20 = agg_value_21;
/* 455 */ }
/* 456 */
/* 457 */ if (!false && (agg_agg_isNull_17_0 ||
/* 458 */ agg_value_20 > agg_expr_2_0)) {
/* 459 */ agg_agg_isNull_17_0 = false;
/* 460 */ agg_value_20 = agg_expr_2_0;
/* 461 */ }
/* 462 */ agg_agg_isNull_20_0 = true;
/* 463 */ int agg_value_23 = -1;
/* 464 */
/* 465 */ boolean agg_isNull_21 =
agg_unsafeRowAggBuffer_0.isNullAt(1);
/* 466 */ int agg_value_24 = agg_isNull_21 ?
/* 467 */ -1 : (agg_unsafeRowAggBuffer_0.getInt(1));
/* 468 */
/* 469 */ if (!agg_isNull_21 && (agg_agg_isNull_20_0 ||
/* 470 */ agg_value_23 > agg_value_24)) {
/* 471 */ agg_agg_isNull_20_0 = false;
/* 472 */ agg_value_23 = agg_value_24;
/* 473 */ }
/* 474 */
/* 475 */ byte agg_caseWhenResultState_0 = -1;
/* 476 */ do {
/* 477 */ boolean agg_value_26 = false;
/* 478 */ agg_value_26 = agg_expr_0_0 > 10;
/* 479 */ if (!false && agg_value_26) {
/* 480 */ agg_caseWhenResultState_0 = (byte)(false ? 1 : 0);
/* 481 */ agg_agg_value_25_0 = agg_expr_0_0;
/* 482 */ continue;
/* 483 */ }
/* 484 */
/* 485 */ agg_caseWhenResultState_0 = (byte)(true ? 1 : 0);
/* 486 */ agg_agg_value_25_0 = -1;
/* 487 */
/* 488 */ } while (false);
/* 489 */ // TRUE if any condition is met and the result is null, or
no any condition is met.
/* 490 */ final boolean agg_isNull_22 = (agg_caseWhenResultState_0 !=
0);
/* 491 */
/* 492 */ if (!agg_isNull_22 && (agg_agg_isNull_20_0 ||
/* 493 */ agg_value_23 > agg_agg_value_25_0)) {
/* 494 */ agg_agg_isNull_20_0 = false;
/* 495 */ agg_value_23 = agg_agg_value_25_0;
/* 496 */ }
/* 497 */ agg_agg_isNull_28_0 = true;
/* 498 */ int agg_value_31 = -1;
/* 499 */
/* 500 */ boolean agg_isNull_29 =
agg_unsafeRowAggBuffer_0.isNullAt(2);
/* 501 */ int agg_value_32 = agg_isNull_29 ?
/* 502 */ -1 : (agg_unsafeRowAggBuffer_0.getInt(2));
/* 503 */
/* 504 */ if (!agg_isNull_29 && (agg_agg_isNull_28_0 ||
/* 505 */ agg_value_31 > agg_value_32)) {
/* 506 */ agg_agg_isNull_28_0 = false;
/* 507 */ agg_value_31 = agg_value_32;
/* 508 */ }
/* 509 */
/* 510 */ if (!false && (agg_agg_isNull_28_0 ||
/* 511 */ agg_value_31 > agg_expr_0_0)) {
/* 512 */ agg_agg_isNull_28_0 = false;
/* 513 */ agg_value_31 = agg_expr_0_0;
/* 514 */ }
/* 515 */ // update unsafe row buffer
/* 516 */ agg_unsafeRowAggBuffer_0.setInt(0, agg_value_20);
/* 517 */
/* 518 */ if (!agg_agg_isNull_20_0) {
/* 519 */ agg_unsafeRowAggBuffer_0.setInt(1, agg_value_23);
/* 520 */ } else {
/* 521 */ agg_unsafeRowAggBuffer_0.setNullAt(1);
/* 522 */ }
/* 523 */
/* 524 */ agg_unsafeRowAggBuffer_0.setInt(2, agg_value_31);
/* 525 */
/* 526 */ }
/* 527 */
/* 528 */ }
......................
/* 554 */ // output the result
/* 555 */
/* 556 */ while (agg_fastHashMapIter_0.next()) {
/* 557 */ UnsafeRow agg_aggKey_0 = (UnsafeRow)
agg_fastHashMapIter_0.getKey();
/* 558 */ UnsafeRow agg_aggBuffer_0 = (UnsafeRow)
agg_fastHashMapIter_0.getValue();
/* 559 */ agg_doAggregateWithKeysOutput_0(agg_aggKey_0,
agg_aggBuffer_0);
/* 560 */
/* 561 */ if (shouldStop()) return;
/* 562 */ }
/* 563 */ agg_fastHashMap_0.close();
/* 564 */
/* 565 */ while (agg_mapIter_0.next()) {
/* 566 */ UnsafeRow agg_aggKey_0 = (UnsafeRow) agg_mapIter_0.getKey();
/* 567 */ UnsafeRow agg_aggBuffer_0 = (UnsafeRow)
agg_mapIter_0.getValue();
/* 568 */ agg_doAggregateWithKeysOutput_0(agg_aggKey_0,
agg_aggBuffer_0);
/* 569 */
/* 570 */ if (shouldStop()) return;
/* 571 */ }
/* 572 */
/* 573 */ agg_mapIter_0.close();
/* 574 */ if (agg_sorter_0 == null) {
/* 575 */ agg_hashMap_0.free();
/* 576 */ }
/* 577 */ }
/* 578 */
/* 579 */ }
whole codegen max code size:954
After modified:
Generated code:
/* 001 */ public Object generate(Object[] references) {
/* 002 */ return new GeneratedIteratorForCodegenStage1(references);
/* 003 */ }
/* 004 */
.............
/* 350 */
/* 351 */ if (agg_fastAggBuffer_0 != null) {
/* 352 */ agg_unsafeRowAggBuffer_0 = agg_fastAggBuffer_0;
/* 353 */ }
/* 354 */
/* 355 */ // common sub-expressions
/* 356 */
/* 357 */ // evaluate aggregate function
/* 358 */ agg_agg_isNull_17_0 = true;
/* 359 */ int agg_value_20 = -1;
/* 360 */
/* 361 */ boolean agg_isNull_18 = agg_unsafeRowAggBuffer_0.isNullAt(0);
/* 362 */ int agg_value_21 = agg_isNull_18 ?
/* 363 */ -1 : (agg_unsafeRowAggBuffer_0.getInt(0));
/* 364 */
/* 365 */ if (!agg_isNull_18 && (agg_agg_isNull_17_0 ||
/* 366 */ agg_value_20 > agg_value_21)) {
/* 367 */ agg_agg_isNull_17_0 = false;
/* 368 */ agg_value_20 = agg_value_21;
/* 369 */ }
/* 370 */
/* 371 */ if (!false && (agg_agg_isNull_17_0 ||
/* 372 */ agg_value_20 > agg_expr_2_0)) {
/* 373 */ agg_agg_isNull_17_0 = false;
/* 374 */ agg_value_20 = agg_expr_2_0;
/* 375 */ }
/* 376 */ agg_agg_isNull_20_0 = true;
/* 377 */ int agg_value_23 = -1;
/* 378 */
/* 379 */ boolean agg_isNull_21 = agg_unsafeRowAggBuffer_0.isNullAt(1);
/* 380 */ int agg_value_24 = agg_isNull_21 ?
/* 381 */ -1 : (agg_unsafeRowAggBuffer_0.getInt(1));
/* 382 */
/* 383 */ if (!agg_isNull_21 && (agg_agg_isNull_20_0 ||
/* 384 */ agg_value_23 > agg_value_24)) {
/* 385 */ agg_agg_isNull_20_0 = false;
/* 386 */ agg_value_23 = agg_value_24;
/* 387 */ }
/* 388 */
/* 389 */ byte agg_caseWhenResultState_0 = -1;
/* 390 */ do {
/* 391 */ boolean agg_value_26 = false;
/* 392 */ agg_value_26 = agg_expr_0_0 > 10;
/* 393 */ if (!false && agg_value_26) {
/* 394 */ agg_caseWhenResultState_0 = (byte)(false ? 1 : 0);
/* 395 */ agg_agg_value_25_0 = agg_expr_0_0;
/* 396 */ continue;
/* 397 */ }
/* 398 */
/* 399 */ agg_caseWhenResultState_0 = (byte)(true ? 1 : 0);
/* 400 */ agg_agg_value_25_0 = -1;
/* 401 */
/* 402 */ } while (false);
/* 403 */ // TRUE if any condition is met and the result is null, or no
any condition is met.
/* 404 */ final boolean agg_isNull_22 = (agg_caseWhenResultState_0 !=
0);
/* 405 */
/* 406 */ if (!agg_isNull_22 && (agg_agg_isNull_20_0 ||
/* 407 */ agg_value_23 > agg_agg_value_25_0)) {
/* 408 */ agg_agg_isNull_20_0 = false;
/* 409 */ agg_value_23 = agg_agg_value_25_0;
/* 410 */ }
/* 411 */ agg_agg_isNull_28_0 = true;
/* 412 */ int agg_value_31 = -1;
/* 413 */
/* 414 */ boolean agg_isNull_29 = agg_unsafeRowAggBuffer_0.isNullAt(2);
/* 415 */ int agg_value_32 = agg_isNull_29 ?
/* 416 */ -1 : (agg_unsafeRowAggBuffer_0.getInt(2));
/* 417 */
/* 418 */ if (!agg_isNull_29 && (agg_agg_isNull_28_0 ||
/* 419 */ agg_value_31 > agg_value_32)) {
/* 420 */ agg_agg_isNull_28_0 = false;
/* 421 */ agg_value_31 = agg_value_32;
/* 422 */ }
/* 423 */
/* 424 */ if (!false && (agg_agg_isNull_28_0 ||
/* 425 */ agg_value_31 > agg_expr_0_0)) {
/* 426 */ agg_agg_isNull_28_0 = false;
/* 427 */ agg_value_31 = agg_expr_0_0;
/* 428 */ }
/* 429 */ // update unsafe row buffer
/* 430 */ agg_unsafeRowAggBuffer_0.setInt(0, agg_value_20);
/* 431 */
/* 432 */ if (!agg_agg_isNull_20_0) {
/* 433 */ agg_unsafeRowAggBuffer_0.setInt(1, agg_value_23);
/* 434 */ } else {
/* 435 */ agg_unsafeRowAggBuffer_0.setNullAt(1);
/* 436 */ }
/* 437 */
/* 438 */ agg_unsafeRowAggBuffer_0.setInt(2, agg_value_31);
/* 439 */
/* 440 */ }
/* 441 */
...........
/* 466 */ // output the result
/* 467 */
/* 468 */ while (agg_fastHashMapIter_0.next()) {
/* 469 */ UnsafeRow agg_aggKey_0 = (UnsafeRow)
agg_fastHashMapIter_0.getKey();
/* 470 */ UnsafeRow agg_aggBuffer_0 = (UnsafeRow)
agg_fastHashMapIter_0.getValue();
/* 471 */ agg_doAggregateWithKeysOutput_0(agg_aggKey_0,
agg_aggBuffer_0);
/* 472 */
/* 473 */ if (shouldStop()) return;
/* 474 */ }
/* 475 */ agg_fastHashMap_0.close();
/* 476 */
/* 477 */ while (agg_mapIter_0.next()) {
/* 478 */ UnsafeRow agg_aggKey_0 = (UnsafeRow) agg_mapIter_0.getKey();
/* 479 */ UnsafeRow agg_aggBuffer_0 = (UnsafeRow)
agg_mapIter_0.getValue();
/* 480 */ agg_doAggregateWithKeysOutput_0(agg_aggKey_0,
agg_aggBuffer_0);
/* 481 */
/* 482 */ if (shouldStop()) return;
/* 483 */ }
/* 484 */
/* 485 */ agg_mapIter_0.close();
/* 486 */ if (agg_sorter_0 == null) {
/* 487 */ agg_hashMap_0.free();
/* 488 */ }
/* 489 */ }
/* 490 */
/* 491 */ }
whole codegen max code size:598
## How was this patch tested?
the Existed test cases.
You can merge this pull request into a Git repository by running:
$ git pull https://github.com/heary-cao/spark fastHashMap
Alternatively you can review and apply these changes as the patch at:
https://github.com/apache/spark/pull/21860.patch
To close this pull request, make a commit to your master/trunk branch
with (at least) the following in the commit message:
This closes #21860
----
commit 137d8b9f279c32cde12f4d488c11be7e697edfab
Author: caoxuewen <cao.xuewen@...>
Date: 2018-07-24T10:45:55Z
Merge the codegen of RegularHashMap and fastHashMap to reduce compiler
maxCodesize when VectorizedHashMap is false
----
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
