ngsg commented on code in PR #4360:
URL: https://github.com/apache/hive/pull/4360#discussion_r1225369948
##########
ql/src/java/org/apache/hadoop/hive/ql/optimizer/SharedWorkOptimizer.java:
##########
@@ -982,15 +943,88 @@ private static void
sharedWorkExtendedOptimization(ParseContext pctx, SharedWork
(Entry<String, TableScanOperator> e) -> e.getValue().getNumChild() ==
0);
}
- /**
- * Obtain the RS input for a mapjoin operator.
- */
- private static ReduceSinkOperator obtainBroadcastInput(MapJoinOperator
mapJoinOp) {
+ @VisibleForTesting
+ public void runMapJoinCacheReuseOptimization(
+ ParseContext pctx, SharedWorkOptimizerCache optimizerCache) throws
SemanticException {
+ // First we group together all the mapjoin operators whose first broadcast
input is the same.
+ final Multimap<Operator<?>, MapJoinOperator> parentToMapJoinOperators =
ArrayListMultimap.create();
+ for (Set<Operator<?>> workOperators : optimizerCache.getWorkGroups()) {
+ for (Operator<?> op : workOperators) {
+ if (op instanceof MapJoinOperator) {
+ MapJoinOperator mapJoinOp = (MapJoinOperator) op;
+ // Only allowed for mapjoin operator
+ if (!mapJoinOp.getConf().isBucketMapJoin() &&
!mapJoinOp.getConf().isDynamicPartitionHashJoin()) {
+ parentToMapJoinOperators.put(
+
obtainFirstBroadcastInput(mapJoinOp).getParentOperators().get(0), mapJoinOp);
+ }
+ }
+ }
+ }
+
+ // For each group, set the cache key accordingly if there is more than one
operator
+ // and input RS operator are equal
+ for (Collection<MapJoinOperator> c :
parentToMapJoinOperators.asMap().values()) {
+ Map<MapJoinOperator, String> mapJoinOpToCacheKey = new HashMap<>();
+ for (MapJoinOperator mapJoinOp : c) {
+ String cacheKey = null;
+ for (Entry<MapJoinOperator, String> e: mapJoinOpToCacheKey.entrySet())
{
+ if (canShareBroadcastInputs(pctx, mapJoinOp, e.getKey())) {
+ cacheKey = e.getValue();
+ break;
+ }
+ }
+
+ if (cacheKey == null) {
+ // Either it is the first map join operator or there was no
equivalent broadcast input,
+ // hence generate cache key
+ cacheKey = MapJoinDesc.generateCacheKey(mapJoinOp.getOperatorId());
+ mapJoinOpToCacheKey.put(mapJoinOp, cacheKey);
+ }
+
+ mapJoinOp.getConf().setCacheKey(cacheKey);
+ }
+ }
+ }
+
+ private ReduceSinkOperator obtainFirstBroadcastInput(MapJoinOperator
mapJoinOp) {
return mapJoinOp.getParentOperators().get(0) instanceof ReduceSinkOperator
?
(ReduceSinkOperator) mapJoinOp.getParentOperators().get(0) :
(ReduceSinkOperator) mapJoinOp.getParentOperators().get(1);
}
+ private boolean canShareBroadcastInputs(
+ ParseContext pctx, MapJoinOperator mapJoinOp1, MapJoinOperator
mapJoinOp2) throws SemanticException {
+ if (mapJoinOp1.getNumParent() != mapJoinOp2.getNumParent()) {
+ return false;
+ }
+
+ if (mapJoinOp1.getConf().getPosBigTable() !=
mapJoinOp2.getConf().getPosBigTable()) {
+ return false;
+ }
+
+ for (int i = 0; i < mapJoinOp1.getNumParent(); i ++) {
+ if (i == mapJoinOp1.getConf().getPosBigTable()) {
+ continue;
+ }
+
+ ReduceSinkOperator parentRS1 = (ReduceSinkOperator)
mapJoinOp1.getParentOperators().get(i);
Review Comment:
MapJoinOperators's parent operator is always ReduceSinkOperator unless it is
at big table position.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
