kasakrisz commented on a change in pull request #1184:
URL: https://github.com/apache/hive/pull/1184#discussion_r460188422
##########
File path:
ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveCardinalityPreservingJoinOptimization.java
##########
@@ -161,127 +174,118 @@ public RelNode trim(RelBuilder relBuilder, RelNode
root) {
return root;
}
- // 4. Collect fields for new Project on the top of Join backs
+ // 3. Join back tables to the top of original plan
Mapping newInputMapping = trimResult.right;
- RexNode[] newProjects = new RexNode[rootFieldList.size()];
- String[] newColumnNames = new String[rootFieldList.size()];
- projectsFromOriginalPlan(rexBuilder,
newInput.getRowType().getFieldCount(), newInput, newInputMapping,
- newProjects, newColumnNames);
+ Map<RexTableInputRef, Integer> tableInputRefMapping = new HashMap<>();
- // 5. Join back tables to the top of original plan
for (TableToJoinBack tableToJoinBack : tableToJoinBackList) {
- LOG.debug("Joining back table " +
tableToJoinBack.projectedFields.relOptHiveTable.getName());
+ LOG.debug("Joining back table " +
tableToJoinBack.joinedBackFields.relOptHiveTable.getName());
- // 5.1 Create new TableScan of tables to join back
- RelOptHiveTable relOptTable =
tableToJoinBack.projectedFields.relOptHiveTable;
+ // 3.1. Create new TableScan of tables to join back
+ RelOptHiveTable relOptTable =
tableToJoinBack.joinedBackFields.relOptHiveTable;
RelOptCluster cluster = relBuilder.getCluster();
HiveTableScan tableScan = new HiveTableScan(cluster,
cluster.traitSetOf(HiveRelNode.CONVENTION),
relOptTable, relOptTable.getHiveTableMD().getTableName(), null,
false, false);
- // 5.2 Project only required fields from this table
+ // 3.2. Create Project with the required fields from this table
RelNode projectTableAccessRel = tableScan.project(
- tableToJoinBack.projectedFields.fieldsInSourceTable, new
HashSet<>(0), REL_BUILDER.get());
+ tableToJoinBack.joinedBackFields.fieldsInSourceTable, new
HashSet<>(0), REL_BUILDER.get());
- Mapping keyMapping = Mappings.create(MappingType.INVERSE_SURJECTION,
- tableScan.getRowType().getFieldCount(),
tableToJoinBack.keys.cardinality());
+ // 3.3. Create mapping between the Project and TableScan
+ Mapping projectMapping =
Mappings.create(MappingType.INVERSE_SURJECTION,
+ tableScan.getRowType().getFieldCount(),
+
tableToJoinBack.joinedBackFields.fieldsInSourceTable.cardinality());
int projectIndex = 0;
+ for (int i : tableToJoinBack.joinedBackFields.fieldsInSourceTable) {
+ projectMapping.set(i, projectIndex);
+ ++projectIndex;
+ }
+
int offset = newInput.getRowType().getFieldCount();
- for (int source : tableToJoinBack.projectedFields.fieldsInSourceTable)
{
- if (tableToJoinBack.keys.get(source)) {
- // 5.3 Map key field to it's index in the Project on the TableScan
- keyMapping.set(source, projectIndex);
- } else {
- // 5.4 if this is not a key field then we need it in the new
Project on the top of Join backs
- ProjectMapping currentProjectMapping =
- tableToJoinBack.projectedFields.mapping.stream()
- .filter(projectMapping ->
projectMapping.indexInSourceTable == source)
- .findFirst().get();
- addToProject(projectTableAccessRel, projectIndex, rexBuilder,
- offset + projectIndex,
- currentProjectMapping.indexInRootProject,
- newProjects, newColumnNames);
+ // 3.4. Map rexTableInputRef to the index where it can be found in the
new Input row type
+ for (TableInputRefHolder mapping :
tableToJoinBack.joinedBackFields.mapping) {
+ int indexInSourceTable = mapping.tableInputRef.getIndex();
+ if (!tableToJoinBack.keys.get(indexInSourceTable)) {
+ // 3.5. if this is not a key field it is shifted by the left input
field count
+ tableInputRefMapping.put(mapping.tableInputRef, offset +
projectMapping.getTarget(indexInSourceTable));
}
- ++projectIndex;
}
- // 5.5 Create Join
+ // 3.7. Create Join
relBuilder.push(newInput);
relBuilder.push(projectTableAccessRel);
RexNode joinCondition = joinCondition(
- newInput, newInputMapping, tableToJoinBack, projectTableAccessRel,
keyMapping, rexBuilder);
+ newInput, newInputMapping, tableToJoinBack, projectTableAccessRel,
projectMapping, rexBuilder);
newInput = relBuilder.join(JoinRelType.INNER, joinCondition).build();
}
- // 6 Create Project on top of all Join backs
+ // 4. Collect rexNodes for Project
+ TableInputRefMapper mapper = new
TableInputRefMapper(tableInputRefMapping, rexBuilder, newInput);
+ List<RexNode> rexNodeList = new
ArrayList<>(rootInput.getRowType().getFieldCount());
+ for (int i = 0; i < rootInput.getRowType().getFieldCount(); i++) {
+ RexNode rexNode = rexNodesToShuttle.get(i);
+ if (rexNode != null) {
+ rexNodeList.add(mapper.apply(rexNode));
+ } else {
+ int target = newInputMapping.getTarget(i);
+ rexNodeList.add(
+
rexBuilder.makeInputRef(newInput.getRowType().getFieldList().get(target).getType(),
target));
+ }
+ }
+
+ // 5. Create Project on top of all Join backs
relBuilder.push(newInput);
- relBuilder.project(asList(newProjects), asList(newColumnNames));
+ relBuilder.project(rexNodeList, newColumnNames);
return root.copy(root.getTraitSet(), singletonList(relBuilder.build()));
} finally {
REL_BUILDER.remove();
}
}
- private List<ProjectedFields> getExpressionLineageOf(
+ private List<JoinedBackFields> getExpressionLineageOf(
List<RexInputRef> projectExpressions, RelNode projectInput) {
RelMetadataQuery relMetadataQuery = RelMetadataQuery.instance();
- Map<RexTableInputRef.RelTableRef, ProjectedFieldsBuilder>
fieldMappingBuilders = new HashMap<>();
+ Map<RexTableInputRef.RelTableRef, JoinedBackFieldsBuilder>
fieldMappingBuilders = new HashMap<>();
List<RexTableInputRef.RelTableRef> tablesOrdered = new ArrayList<>(); //
use this list to keep the order of tables
for (RexInputRef expr : projectExpressions) {
Set<RexNode> expressionLineage =
relMetadataQuery.getExpressionLineage(projectInput, expr);
if (expressionLineage == null || expressionLineage.size() != 1) {
- LOG.debug("Lineage can not be determined of expression: " + expr);
+ LOG.debug("Lineage of expression can not be determined: " + expr);
Review comment:
fixed
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
