This is an automated email from the ASF dual-hosted git repository. xxyu pushed a commit to branch kylin5 in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 82516523250251fbf6ed6a1570eb3789b54b7e13 Author: songzhxlh-max <59271430+songzhxlh-...@users.noreply.github.com> AuthorDate: Fri Oct 21 15:07:28 2022 +0800 KYLIN-5332 Query select star expose column from dataflow --- .../metadata/cube/cuboid/NLayoutCandidate.java | 4 ++ .../metadata/cube/cuboid/NQueryLayoutChooser.java | 7 +-- .../cube/model/NDataflowCapabilityChecker.java | 1 + .../org/apache/kylin/query/routing/Candidate.java | 62 ++++++++++++++-------- .../kylin/query/routing/RealizationChooser.java | 14 ++++- 5 files changed, 62 insertions(+), 26 deletions(-) diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/cuboid/NLayoutCandidate.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/cuboid/NLayoutCandidate.java index a7ceda70a2..caef168503 100644 --- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/cuboid/NLayoutCandidate.java +++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/cuboid/NLayoutCandidate.java @@ -42,6 +42,10 @@ public class NLayoutCandidate implements IRealizationCandidate { @Setter private double cost; + @Getter + @Setter + private int layoutUnmatchedColsSize; + @Setter @Getter private CapabilityResult capabilityResult; diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/cuboid/NQueryLayoutChooser.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/cuboid/NQueryLayoutChooser.java index 0b32223e33..ccada17bd7 100644 --- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/cuboid/NQueryLayoutChooser.java +++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/cuboid/NQueryLayoutChooser.java @@ -110,7 +110,6 @@ public class NQueryLayoutChooser { } val projectInstance = NProjectManager.getInstance(KylinConfig.getInstanceFromEnv()) .getProject(dataflow.getProject()); - double influenceFactor = 1.0; for (NDataLayout dataLayout : commonLayouts) { log.trace("Matching layout {}", dataLayout); CapabilityResult tempResult = new CapabilityResult(); @@ -119,18 +118,20 @@ public class NQueryLayoutChooser { LayoutEntity layout = indexPlan.getLayoutEntity(dataLayout.getLayoutId()); log.trace("Matching indexEntity {}", indexEntity); + NLayoutCandidate candidate = new NLayoutCandidate(layout); var matchResult = tableIndexMatcher.match(layout); + double influenceFactor = 1.0; if (!matchResult.isMatched()) { matchResult = aggIndexMatcher.match(layout); } else if (projectInstance.getConfig().useTableIndexAnswerSelectStarEnabled()) { - influenceFactor += influenceFactor + tableIndexMatcher.getLayoutUnmatchedColsSize(); + influenceFactor += tableIndexMatcher.getLayoutUnmatchedColsSize(); + candidate.setLayoutUnmatchedColsSize(tableIndexMatcher.getLayoutUnmatchedColsSize()); } if (!matchResult.isMatched()) { log.trace("Matching failed"); continue; } - NLayoutCandidate candidate = new NLayoutCandidate(layout); tempResult.influences = matchResult.getInfluences(); candidate.setCost(dataLayout.getRows() * (tempResult.influences.size() + influenceFactor)); if (!matchResult.getNeedDerive().isEmpty()) { diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/model/NDataflowCapabilityChecker.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/model/NDataflowCapabilityChecker.java index 0968253dbe..34d3a016f8 100644 --- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/model/NDataflowCapabilityChecker.java +++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/model/NDataflowCapabilityChecker.java @@ -85,6 +85,7 @@ public class NDataflowCapabilityChecker { if (candidateAndInfluence != null) { chosenCandidate = candidateAndInfluence; result.influences.addAll(candidateAndInfluence.getCapabilityResult().influences); + result.setLayoutUnmatchedColsSize(candidateAndInfluence.getLayoutUnmatchedColsSize()); logger.info("Matched layout {} snapshot in dataflow {} ", chosenCandidate, dataflow); } } diff --git a/src/query-common/src/main/java/org/apache/kylin/query/routing/Candidate.java b/src/query-common/src/main/java/org/apache/kylin/query/routing/Candidate.java index c8eb9a2247..06ad7c2b65 100644 --- a/src/query-common/src/main/java/org/apache/kylin/query/routing/Candidate.java +++ b/src/query-common/src/main/java/org/apache/kylin/query/routing/Candidate.java @@ -39,6 +39,7 @@ import lombok.Setter; public class Candidate { public static final CandidateComparator COMPARATOR = new CandidateComparator(); + public static final CandidateTableIndexComparator COMPARATOR_TABLE_INDEX = new CandidateTableIndexComparator(); // ============================================================================ @@ -109,36 +110,53 @@ public class Candidate { @Override public int compare(Candidate c1, Candidate c2) { - IRealization real1 = c1.getRealization(); - IRealization real2 = c2.getRealization(); - - if (QueryContext.current().getModelPriorities().length > 0) { - - Map<String, Integer> priorities = new HashMap<>(); - for (int i = 0; i < QueryContext.current().getModelPriorities().length; i++) { - priorities.put(QueryContext.current().getModelPriorities()[i], i); - } - - int comp = priorities.getOrDefault(real1.getModel().getAlias().toUpperCase(Locale.ROOT), - Integer.MAX_VALUE) - - priorities.getOrDefault(real2.getModel().getAlias().toUpperCase(Locale.ROOT), - Integer.MAX_VALUE); - if (comp != 0) { - return comp; - } + return compareCandidate(c1, c2); + } + } + + public static class CandidateTableIndexComparator implements Comparator<Candidate> { + + @Override + public int compare(Candidate c1, Candidate c2) { + CapabilityResult capabilityResult1 = c1.getCapability(); + CapabilityResult capabilityResult2 = c2.getCapability(); + if (capabilityResult1.getLayoutUnmatchedColsSize() != capabilityResult2.getLayoutUnmatchedColsSize()) { + return capabilityResult1.getLayoutUnmatchedColsSize() - capabilityResult2.getLayoutUnmatchedColsSize(); } + return compareCandidate(c1, c2); + } + } - int comp = real1.getCost() - real2.getCost(); - if (comp != 0) { - return comp; + private static int compareCandidate(Candidate c1, Candidate c2) { + IRealization real1 = c1.getRealization(); + IRealization real2 = c2.getRealization(); + + if (QueryContext.current().getModelPriorities().length > 0) { + + Map<String, Integer> priorities = new HashMap<>(); + for (int i = 0; i < QueryContext.current().getModelPriorities().length; i++) { + priorities.put(QueryContext.current().getModelPriorities()[i], i); } - comp = Double.compare(c1.capability.cost, c2.capability.cost); + int comp = priorities.getOrDefault(real1.getModel().getAlias().toUpperCase(Locale.ROOT), + Integer.MAX_VALUE) + - priorities.getOrDefault(real2.getModel().getAlias().toUpperCase(Locale.ROOT), + Integer.MAX_VALUE); if (comp != 0) { return comp; } + } + + int comp = real1.getCost() - real2.getCost(); + if (comp != 0) { + return comp; + } - return real1.getModel().getId().compareTo(real2.getModel().getId()); + comp = Double.compare(c1.capability.cost, c2.capability.cost); + if (comp != 0) { + return comp; } + + return real1.getModel().getId().compareTo(real2.getModel().getId()); } } diff --git a/src/query-common/src/main/java/org/apache/kylin/query/routing/RealizationChooser.java b/src/query-common/src/main/java/org/apache/kylin/query/routing/RealizationChooser.java index b27d486e55..284a2fc190 100644 --- a/src/query-common/src/main/java/org/apache/kylin/query/routing/RealizationChooser.java +++ b/src/query-common/src/main/java/org/apache/kylin/query/routing/RealizationChooser.java @@ -91,6 +91,7 @@ import org.apache.kylin.metadata.model.TableRef; import org.apache.kylin.metadata.model.TblColRef; import org.apache.kylin.metadata.project.NProjectLoader; import org.apache.kylin.metadata.project.NProjectManager; +import org.apache.kylin.metadata.project.ProjectInstance; import org.apache.kylin.metadata.realization.CapabilityResult; import org.apache.kylin.metadata.realization.IRealization; import org.apache.kylin.metadata.realization.NoRealizationFoundException; @@ -243,7 +244,7 @@ public class RealizationChooser { } // Step 3. find the lowest-cost candidate - candidates.sort(Candidate.COMPARATOR); + sortCandidate(context, candidates); logger.trace("Cost Sorted Realizations {}", candidates); if (!candidates.isEmpty()) { Candidate selectedCandidate = candidates.get(0); @@ -273,6 +274,17 @@ public class RealizationChooser { throw new NoRealizationFoundException("No realization found for " + toErrorMsg(context)); } + private static void sortCandidate(OLAPContext context, List<Candidate> candidates) { + ProjectInstance projectInstance = NProjectManager.getInstance(KylinConfig.getInstanceFromEnv()) + .getProject(context.olapSchema.getProjectName()); + if (projectInstance.getConfig().useTableIndexAnswerSelectStarEnabled() + && context.getSQLDigest().isRawQuery) { + candidates.sort(Candidate.COMPARATOR_TABLE_INDEX); + } else { + candidates.sort(Candidate.COMPARATOR); + } + } + private static void checkNoRealizationWithStreaming(OLAPContext context) { String projectName = context.olapSchema.getProjectName(); KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();