This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch kylin-on-parquet-v2
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 7a336fab3eb9c5bb931eba031f9249f334fad066
Author: Zhichao Zhang <441586...@qq.com>
AuthorDate: Tue Aug 25 16:01:17 2020 +0800
KYLIN-4660 Remove duplicated/misleading code or comment
---
.../java/org/apache/kylin/common/KylinConfig.java | 24 ++++++++++-----------
.../org/apache/kylin/common/KylinConfigBase.java | 1 +
.../java/org/apache/kylin/cube/CubeSegment.java | 2 +-
.../kylin/job/execution/AbstractExecutable.java | 5 +++--
.../apache/kylin/job/execution/JobTypeEnum.java | 25 ----------------------
.../metadata/realization/RealizationType.java | 2 +-
.../apache/kylin/engine/spark/job/JobStepType.java | 4 +---
.../kylin/engine/spark/job/NSparkCubingUtil.java | 4 ++--
.../kylin/engine/spark/job/NSparkMergingJob.java | 3 +--
.../merger/AfterMergeOrRefreshResourceMerger.java | 4 ++--
.../engine/spark/builder/CreateFlatTable.scala | 2 +-
.../kylin/engine/spark/job/CubeBuildJob.java | 7 +++---
.../org/apache/kylin/query/routing/Candidate.java | 1 -
13 files changed, 29 insertions(+), 55 deletions(-)
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
b/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
index 8562314..c18c643 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
@@ -587,23 +587,23 @@ public class KylinConfig extends KylinConfigBase {
throw new IllegalArgumentException(localMetaDir + " is not a
valid local meta dir");
destroyInstance();
- String canonicalPath = localMetaDir;
- try {
- // remove the ".." in path string
- canonicalPath = new File(localMetaDir).getCanonicalPath();
- } catch (IOException e) {
- throw new IllegalStateException("");
- }
- logger.info("Setting KylinConfig to {} in UT.", canonicalPath);
- System.setProperty(KylinConfig.KYLIN_CONF, canonicalPath);
+ logger.info("Setting KylinConfig to {} in UT.", localMetaDir);
+ System.setProperty(KylinConfig.KYLIN_CONF, localMetaDir);
KylinConfig config = KylinConfig.getInstanceFromEnv();
- config.setMetadataUrl(canonicalPath);
+ config.setMetadataUrl(localMetaDir);
- File workingDir = new File(canonicalPath, "working-dir");
+ // create a local working dir
+ File workingDir = new File(localMetaDir, "working-dir");
workingDir.mkdirs();
- String path = workingDir.getAbsolutePath();
+ String path = "";
+ try {
+ // remove the ".." in path string
+ path = workingDir.getCanonicalPath();
+ } catch (IOException e) {
+ throw new IllegalStateException("");
+ }
if (!path.startsWith("/"))
path = "/" + path;
if (!path.endsWith("/"))
diff --git
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index ac93fd4..78ba123 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -307,6 +307,7 @@ public abstract class KylinConfigBase implements
Serializable {
String metaId = getMetadataUrlPrefix().replace(':', '-');
//transform relative path for local metadata
if (metaId.startsWith("../")) {
+ metaId = metaId.replace("../", "");
metaId = metaId.replace('/', '-');
}
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
b/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
index 70fb501..f7072cc 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
@@ -69,7 +69,7 @@ public class CubeSegment implements IBuildable, ISegment,
Serializable {
@JsonProperty("name")
private String name;
@JsonProperty("storage_location_identifier")
- private String storageLocationIdentifier; // HTable name
+ private String storageLocationIdentifier;
@JsonProperty("date_range_start")
private long dateRangeStart;
@JsonProperty("date_range_end")
diff --git
a/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
b/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
index 86ac294..908e154 100644
---
a/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
+++
b/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
@@ -38,6 +38,7 @@ import org.apache.kylin.common.util.RandomUtil;
import org.apache.kylin.common.util.StringUtil;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
+import org.apache.kylin.cube.model.CubeBuildTypeEnum;
import org.apache.kylin.job.exception.ExecuteException;
import org.apache.kylin.job.exception.JobStoppedException;
import org.apache.kylin.job.exception.PersistentException;
@@ -81,7 +82,7 @@ public abstract class AbstractExecutable implements
Executable, Idempotent {
private AbstractExecutable parentExecutable = null;
private Map<String, String> params = Maps.newHashMap();
protected Integer priority;
- private JobTypeEnum jobType;
+ private CubeBuildTypeEnum jobType;
protected String project;
private String targetSubject;
private List<String> targetSegments = Lists.newArrayList();//uuid of
related segments
@@ -592,7 +593,7 @@ public abstract class AbstractExecutable implements
Executable, Idempotent {
this.project = project;
}
- public void setJobType(JobTypeEnum jobType) {
+ public void setJobType(CubeBuildTypeEnum jobType) {
this.jobType = jobType;
}
diff --git
a/core-job/src/main/java/org/apache/kylin/job/execution/JobTypeEnum.java
b/core-job/src/main/java/org/apache/kylin/job/execution/JobTypeEnum.java
deleted file mode 100644
index 2823431..0000000
--- a/core-job/src/main/java/org/apache/kylin/job/execution/JobTypeEnum.java
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.execution;
-
-public enum JobTypeEnum {
- INDEX_REFRESH, INDEX_MERGE, INDEX_BUILD, INC_BUILD,
-
- TABLE_SAMPLING
-}
diff --git
a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/RealizationType.java
b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/RealizationType.java
index 3fcfe63..8b9defc 100644
---
a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/RealizationType.java
+++
b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/RealizationType.java
@@ -23,5 +23,5 @@ package org.apache.kylin.metadata.realization;
//TODO: change to String for plugin
public enum RealizationType {
- CUBE, INVERTED_INDEX, HYBRID
+ CUBE, HYBRID
}
diff --git
a/kylin-spark-project/kylin-spark-engine/src/main/java/org/apache/kylin/engine/spark/job/JobStepType.java
b/kylin-spark-project/kylin-spark-engine/src/main/java/org/apache/kylin/engine/spark/job/JobStepType.java
index a24a265..a81312a 100644
---
a/kylin-spark-project/kylin-spark-engine/src/main/java/org/apache/kylin/engine/spark/job/JobStepType.java
+++
b/kylin-spark-project/kylin-spark-engine/src/main/java/org/apache/kylin/engine/spark/job/JobStepType.java
@@ -21,7 +21,5 @@ package org.apache.kylin.engine.spark.job;
public enum JobStepType {
RESOURCE_DETECT,
- CLEAN_UP_AFTER_MERGE, CUBING, MERGING,
-
- SAMPLING
+ CLEAN_UP_AFTER_MERGE, CUBING, MERGING
}
diff --git
a/kylin-spark-project/kylin-spark-engine/src/main/java/org/apache/kylin/engine/spark/job/NSparkCubingUtil.java
b/kylin-spark-project/kylin-spark-engine/src/main/java/org/apache/kylin/engine/spark/job/NSparkCubingUtil.java
index f80e9cf..5438896 100644
---
a/kylin-spark-project/kylin-spark-engine/src/main/java/org/apache/kylin/engine/spark/job/NSparkCubingUtil.java
+++
b/kylin-spark-project/kylin-spark-engine/src/main/java/org/apache/kylin/engine/spark/job/NSparkCubingUtil.java
@@ -87,8 +87,8 @@ public class NSparkCubingUtil {
public static String getStoragePath(CubeSegment nDataSegment, Long
layoutId) {
String hdfsWorkingDir =
nDataSegment.getConfig().getReadHdfsWorkingDirectory();
- return hdfsWorkingDir +
- getStoragePathWithoutPrefix(nDataSegment.getProject(),
nDataSegment.getCubeInstance().getId(), nDataSegment.getUuid(), layoutId);
+ return hdfsWorkingDir +
getStoragePathWithoutPrefix(nDataSegment.getProject(),
+ nDataSegment.getCubeInstance().getId(),
nDataSegment.getUuid(), layoutId);
}
static Set<String> toSegmentNames(Segments<CubeSegment> segments) {
diff --git
a/kylin-spark-project/kylin-spark-engine/src/main/java/org/apache/kylin/engine/spark/job/NSparkMergingJob.java
b/kylin-spark-project/kylin-spark-engine/src/main/java/org/apache/kylin/engine/spark/job/NSparkMergingJob.java
index 25de377..7efc946 100644
---
a/kylin-spark-project/kylin-spark-engine/src/main/java/org/apache/kylin/engine/spark/job/NSparkMergingJob.java
+++
b/kylin-spark-project/kylin-spark-engine/src/main/java/org/apache/kylin/engine/spark/job/NSparkMergingJob.java
@@ -32,7 +32,6 @@ import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.CubeSegment;
-import org.apache.kylin.job.execution.JobTypeEnum;
import org.apache.kylin.metadata.MetadataConstants;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -82,7 +81,7 @@ public class NSparkMergingJob extends CubingJob {
job.setParam(MetadataConstants.P_DATA_RANGE_START,
mergedSegment.getSegRange().start.toString());
job.setParam(MetadataConstants.P_DATA_RANGE_END,
mergedSegment.getSegRange().end.toString());
job.setParam(MetadataConstants.P_OUTPUT_META_URL,
cube.getConfig().getMetadataUrl().toString());
- job.setParam(MetadataConstants.P_JOB_TYPE,
String.valueOf(JobTypeEnum.INDEX_MERGE));
+ job.setParam(MetadataConstants.P_JOB_TYPE, String.valueOf(jobType));
job.setParam(MetadataConstants.P_CUBOID_NUMBER,
String.valueOf(cube.getDescriptor().getAllCuboids().size()));
JobStepFactory.addStep(job, JobStepType.RESOURCE_DETECT, cube);
diff --git
a/kylin-spark-project/kylin-spark-engine/src/main/java/org/apache/kylin/engine/spark/merger/AfterMergeOrRefreshResourceMerger.java
b/kylin-spark-project/kylin-spark-engine/src/main/java/org/apache/kylin/engine/spark/merger/AfterMergeOrRefreshResourceMerger.java
index f634917..2fb1812 100644
---
a/kylin-spark-project/kylin-spark-engine/src/main/java/org/apache/kylin/engine/spark/merger/AfterMergeOrRefreshResourceMerger.java
+++
b/kylin-spark-project/kylin-spark-engine/src/main/java/org/apache/kylin/engine/spark/merger/AfterMergeOrRefreshResourceMerger.java
@@ -30,9 +30,9 @@ import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.cube.CubeUpdate;
+import org.apache.kylin.cube.model.CubeBuildTypeEnum;
import org.apache.kylin.engine.mr.steps.CubingExecutableUtil;
import org.apache.kylin.job.execution.AbstractExecutable;
-import org.apache.kylin.job.execution.JobTypeEnum;
import org.apache.kylin.metadata.MetadataConstants;
import org.apache.kylin.metadata.model.IStorageAware;
import org.apache.kylin.metadata.model.SegmentStatusEnum;
@@ -69,7 +69,7 @@ public class AfterMergeOrRefreshResourceMerger extends
MetadataMerger {
Collections.sort(toRemoveSegments);
makeSnapshotForNewSegment(mergedSegment, toRemoveSegments);
- if (String.valueOf(JobTypeEnum.INDEX_MERGE).equals(jobType)) {
+ if (String.valueOf(CubeBuildTypeEnum.MERGE).equals(jobType)) {
Optional<Long> reduce =
toRemoveSegments.stream().map(CubeSegment::getSizeKB).filter(size -> size != -1)
.reduce(Long::sum);
Optional<Long> inputRecords =
toRemoveSegments.stream().map(CubeSegment::getInputRecords).filter(records ->
records != -1)
diff --git
a/kylin-spark-project/kylin-spark-engine/src/main/scala/org/apache/kylin/engine/spark/builder/CreateFlatTable.scala
b/kylin-spark-project/kylin-spark-engine/src/main/scala/org/apache/kylin/engine/spark/builder/CreateFlatTable.scala
index 8d11974..dd383c3 100644
---
a/kylin-spark-project/kylin-spark-engine/src/main/scala/org/apache/kylin/engine/spark/builder/CreateFlatTable.scala
+++
b/kylin-spark-project/kylin-spark-engine/src/main/scala/org/apache/kylin/engine/spark/builder/CreateFlatTable.scala
@@ -170,7 +170,7 @@ object CreateFlatTable extends Logging {
ss: SparkSession): Dataset[Row] = {
var afterJoin = rootFactDataset
val joinType = joinDesc.joinType
- if (joinType != null && !StringUtils.isEmpty(joinType)) {
+ if (StringUtils.isNotEmpty(joinType)) {
val pk = joinDesc.PKS
val fk = joinDesc.FKS
if (pk.length != fk.length) {
diff --git
a/kylin-spark-project/kylin-spark-engine/src/main/scala/org/apache/kylin/engine/spark/job/CubeBuildJob.java
b/kylin-spark-project/kylin-spark-engine/src/main/scala/org/apache/kylin/engine/spark/job/CubeBuildJob.java
index d8788cb..80b0674 100644
---
a/kylin-spark-project/kylin-spark-engine/src/main/scala/org/apache/kylin/engine/spark/job/CubeBuildJob.java
+++
b/kylin-spark-project/kylin-spark-engine/src/main/scala/org/apache/kylin/engine/spark/job/CubeBuildJob.java
@@ -222,12 +222,13 @@ public class CubeBuildJob extends SparkApplication {
buildLayoutWithUpdate.submit(new
BuildLayoutWithUpdate.JobEntity() {
@Override
public String getName() {
- return "build-index-" + index.getId();
+ return "build-cuboid-" + index.getId();
}
@Override
public LayoutEntity build() throws IOException {
- return buildIndex(seg, index, parentDS, st,
info.getLayoutId(), parentDSCnt);
+ return buildCuboid(seg, index, parentDS, st,
info.getLayoutId(),
+ parentDSCnt);
}
}, config);
allIndexesInCurrentLayer.add(index);
@@ -288,7 +289,7 @@ public class CubeBuildJob extends SparkApplication {
return ResourceDetectUtils.selectMaxValueInFiles(fileStatuses);
}
- private LayoutEntity buildIndex(SegmentInfo seg, LayoutEntity cuboid,
Dataset<Row> parent,
+ private LayoutEntity buildCuboid(SegmentInfo seg, LayoutEntity cuboid,
Dataset<Row> parent,
SpanningTree spanningTree, long parentId,
long parentDSCnt) throws IOException {
String parentName = String.valueOf(parentId);
if (parentId == ParentSourceChooser.FLAT_TABLE_FLAG()) {
diff --git a/query/src/main/java/org/apache/kylin/query/routing/Candidate.java
b/query/src/main/java/org/apache/kylin/query/routing/Candidate.java
index bc17721..ddf5d0d 100644
--- a/query/src/main/java/org/apache/kylin/query/routing/Candidate.java
+++ b/query/src/main/java/org/apache/kylin/query/routing/Candidate.java
@@ -36,7 +36,6 @@ public class Candidate implements Comparable<Candidate> {
static {
DEFAULT_PRIORITIES.put(RealizationType.HYBRID, 0);
DEFAULT_PRIORITIES.put(RealizationType.CUBE, 1);
- DEFAULT_PRIORITIES.put(RealizationType.INVERTED_INDEX, 1);
}
/** for test only */
