http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/cube/src/main/java/org/apache/kylin/cube/CubeDescUpgrader.java ---------------------------------------------------------------------- diff --git a/cube/src/main/java/org/apache/kylin/cube/CubeDescUpgrader.java b/cube/src/main/java/org/apache/kylin/cube/CubeDescUpgrader.java deleted file mode 100644 index 7420912..0000000 --- a/cube/src/main/java/org/apache/kylin/cube/CubeDescUpgrader.java +++ /dev/null @@ -1,308 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.cube; - -import java.io.IOException; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.UUID; - -import org.apache.commons.lang.ArrayUtils; -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.kylin.common.KylinConfig; -import org.apache.kylin.common.persistence.JsonSerializer; -import org.apache.kylin.common.persistence.ResourceStore; -import org.apache.kylin.common.persistence.Serializer; -import org.apache.kylin.cube.model.DimensionDesc; -import org.apache.kylin.cube.model.HierarchyDesc; -import org.apache.kylin.cube.model.RowKeyColDesc; -import org.apache.kylin.cube.model.RowKeyDesc; -import org.apache.kylin.cube.model.v1.CubeDesc; -import org.apache.kylin.cube.model.v1.CubePartitionDesc; -import org.apache.kylin.metadata.MetadataManager; -import org.apache.kylin.metadata.model.DataModelDesc; -import org.apache.kylin.metadata.model.JoinDesc; -import org.apache.kylin.metadata.model.LookupDesc; -import org.apache.kylin.metadata.model.PartitionDesc; -import org.apache.kylin.metadata.model.TableDesc; - -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; - -public class CubeDescUpgrader { - - private String resourcePath; - - @SuppressWarnings("unused") - private static final Log logger = LogFactory.getLog(CubeDescUpgrader.class); - - private static final Serializer<CubeDesc> CUBE_DESC_SERIALIZER_V1 = new JsonSerializer<CubeDesc>(CubeDesc.class); - - public CubeDescUpgrader(String resourcePath) { - this.resourcePath = resourcePath; - } - - public org.apache.kylin.cube.model.CubeDesc upgrade() throws IOException { - CubeDesc oldModel = loadOldCubeDesc(resourcePath); - - org.apache.kylin.cube.model.CubeDesc newModel = new org.apache.kylin.cube.model.CubeDesc(); - - copyUnChangedProperties(oldModel, newModel); - - DataModelDesc model = extractDataModel(oldModel, newModel); - newModel.setModel(model); - - updateDimensions(oldModel, newModel); - - updateRowkeyDictionary(newModel); - - return newModel; - } - - private void updateRowkeyDictionary(org.apache.kylin.cube.model.CubeDesc newModel) { - - DataModelDesc modelDesc = newModel.getModel(); - Map<String, String> pkToFK = Maps.newHashMap(); - for (LookupDesc lookupDesc : modelDesc.getLookups()) { - if (lookupDesc.getJoin() != null) { - JoinDesc join = lookupDesc.getJoin(); - for (int i = 0; i < join.getForeignKey().length; i++) { - pkToFK.put(join.getPrimaryKey()[i], join.getForeignKey()[i]); - } - } - } - RowKeyDesc rowKey = newModel.getRowkey(); - - for (RowKeyColDesc rowkeyCol : rowKey.getRowKeyColumns()) { - if (rowkeyCol.getDictionary() != null && rowkeyCol.getDictionary().length() > 0) - rowkeyCol.setDictionary("true"); - - if (pkToFK.containsKey(rowkeyCol.getColumn())) { - rowkeyCol.setColumn(pkToFK.get(rowkeyCol.getColumn())); - } - } - - for (String[] aggregationGroup : rowKey.getAggregationGroups()) { - for (int i = 0; i < aggregationGroup.length; i++) { - if (pkToFK.containsKey(aggregationGroup[i])) { - aggregationGroup[i] = pkToFK.get(aggregationGroup[i]); - } - } - } - - } - - private void copyUnChangedProperties(CubeDesc oldModel, org.apache.kylin.cube.model.CubeDesc newModel) { - - newModel.setUuid(oldModel.getUuid()); - newModel.setName(oldModel.getName()); - newModel.setDescription(oldModel.getDescription()); - newModel.setNullStrings(oldModel.getNullStrings()); - - newModel.setMeasures(oldModel.getMeasures()); - newModel.setRowkey(oldModel.getRowkey()); - newModel.setHbaseMapping(oldModel.getHBaseMapping()); - - newModel.setSignature(oldModel.getSignature()); - - newModel.setNotifyList(oldModel.getNotifyList()); - newModel.setLastModified(oldModel.getLastModified()); - } - - private DimensionDesc newDimensionDesc(org.apache.kylin.cube.model.v1.DimensionDesc dim, int dimId, String name) { - DimensionDesc newDim = new DimensionDesc(); - - newDim.setId(dimId); - newDim.setName(name); - newDim.setTable(getMetadataManager().appendDBName(dim.getTable())); - - return newDim; - } - - private void updateDimensions(CubeDesc oldModel, org.apache.kylin.cube.model.CubeDesc newModel) { - List<org.apache.kylin.cube.model.v1.DimensionDesc> oldDimensions = oldModel.getDimensions(); - - List<DimensionDesc> newDimensions = Lists.newArrayList(); - newModel.setDimensions(newDimensions); - - int dimId = 0; - for (org.apache.kylin.cube.model.v1.DimensionDesc dim : oldDimensions) { - - DimensionDesc newDim = null; - // if a dimension defines "column", "derived" and "hierarchy" at the same time, separate it into three dimensions; - - boolean needNameSuffix = false; - if (dim.getColumn() != null && !"{FK}".equals(dim.getColumn())) { - //column on fact table - newDim = newDimensionDesc(dim, dimId++, dim.getName()); - newDimensions.add(newDim); - newDim.setColumn(new String[] { dim.getColumn() }); - needNameSuffix = true; - } else if (ArrayUtils.isEmpty(dim.getDerived()) && ArrayUtils.isEmpty(dim.getHierarchy())) { - // user defines a lookup table, but didn't use any column other than the pk, in this case, convert to use fact table's fk - newDim = newDimensionDesc(dim, dimId++, dim.getName()); - newDimensions.add(newDim); - newDim.setTable(getMetadataManager().appendDBName(newModel.getFactTable())); - - newDim.setColumn(dim.getJoin().getForeignKey()); - } - - if (!ArrayUtils.isEmpty(dim.getDerived())) { - newDim = newDimensionDesc(dim, dimId++, dim.getName() + (needNameSuffix ? "_DERIVED" : "")); - newDimensions.add(newDim); - newDim.setDerived(dim.getDerived()); - newDim.setColumn(null); // derived column must come from a lookup table; in this case the fk will be the dimension column, no need to explicitly declare it; - needNameSuffix = true; - } - - if (!ArrayUtils.isEmpty(dim.getHierarchy())) { - newDim = newDimensionDesc(dim, dimId++, dim.getName() + (needNameSuffix ? "_HIERARCHY" : "")); - newDimensions.add(newDim); - - newDim.setHierarchy(true); - - List<String> columns = Lists.newArrayList(); - for (HierarchyDesc hierarch : dim.getHierarchy()) { - String col = hierarch.getColumn(); - columns.add(col); - } - - newDim.setColumn(columns.toArray(new String[columns.size()])); - } - - } - } - - private DataModelDesc extractDataModel(CubeDesc oldModel, org.apache.kylin.cube.model.CubeDesc newModel) { - - DataModelDesc dm = new DataModelDesc(); - dm.setUuid(UUID.randomUUID().toString()); - String factTable = oldModel.getFactTable(); - dm.setName(oldModel.getName()); - dm.setFactTable(getMetadataManager().appendDBName(factTable)); - - newModel.setModelName(dm.getName()); - - List<org.apache.kylin.cube.model.v1.DimensionDesc> oldDimensions = oldModel.getDimensions(); - - List<LookupDesc> lookups = Lists.newArrayList(); - List<String> lookupTables = Lists.newArrayList(); - for (org.apache.kylin.cube.model.v1.DimensionDesc dim : oldDimensions) { - JoinDesc join = dim.getJoin(); - if (join != null && !StringUtils.isEmpty(join.getType()) && join.getForeignKey() != null && join.getForeignKey().length > 0) { - String table = dim.getTable(); - table = getMetadataManager().appendDBName(table); - - if (lookupTables.contains(table)) { - continue; - } - LookupDesc lookup = new LookupDesc(); - lookup.setJoin(join); - lookup.setTable(table); - lookupTables.add(table); - lookups.add(lookup); - } - } - - dm.setLookups(lookups.toArray(new LookupDesc[lookups.size()])); - dm.setFilterCondition(oldModel.getFilterCondition()); - updatePartitionDesc(oldModel, dm); - - if (oldModel.getCapacity() == CubeDesc.CubeCapacity.SMALL) { - dm.setCapacity(DataModelDesc.RealizationCapacity.SMALL); - } else if (oldModel.getCapacity() == CubeDesc.CubeCapacity.MEDIUM) { - dm.setCapacity(DataModelDesc.RealizationCapacity.MEDIUM); - } else if (oldModel.getCapacity() == CubeDesc.CubeCapacity.LARGE) { - dm.setCapacity(DataModelDesc.RealizationCapacity.LARGE); - } - - return dm; - } - - private void updatePartitionDesc(CubeDesc oldModel, DataModelDesc dm) { - - CubePartitionDesc partition = oldModel.getCubePartitionDesc(); - PartitionDesc newPartition = new PartitionDesc(); - - if (partition.getPartitionDateColumn() != null) { - String partitionCol = partition.getPartitionDateColumn(); - - String[] tablecolumn = partitionCol.split("\\."); - if (tablecolumn != null && tablecolumn.length == 2) { - // pattern is <tablename>.<colname> - String tableFullName = getMetadataManager().appendDBName(tablecolumn[0]); - newPartition.setPartitionDateColumn(tableFullName + "." + tablecolumn[1]); - } else { - - if (partitionCol.indexOf(".") < 0) { - // pattern is <colname> - partitionCol = dm.getFactTable() + "." + partitionCol; - } - - newPartition.setPartitionDateColumn(partitionCol); - } - } - - // only append is supported - newPartition.setCubePartitionType(PartitionDesc.PartitionType.APPEND); - - newPartition.setPartitionDateStart(partition.getPartitionDateStart()); - - dm.setPartitionDesc(newPartition); - } - - private CubeDesc loadOldCubeDesc(String path) throws IOException { - ResourceStore store = getStore(); - - CubeDesc ndesc = store.getResource(path, CubeDesc.class, CUBE_DESC_SERIALIZER_V1); - - if (StringUtils.isBlank(ndesc.getName())) { - throw new IllegalStateException("CubeDesc name must not be blank"); - } - - Map<String, TableDesc> tableMap = getMetadataManager().getAllTablesMap(); - Map<String, TableDesc> newMap = Maps.newHashMap(); - for (Entry<String, TableDesc> entry : tableMap.entrySet()) { - String t = entry.getKey(); - - if (t.indexOf(".") > 0) { - newMap.put(t.substring(t.indexOf(".") + 1), entry.getValue()); - - } - } - ndesc.init(KylinConfig.getInstanceFromEnv(), newMap); - - if (ndesc.getError().isEmpty() == false) { - throw new IllegalStateException("Cube desc at " + path + " has issues: " + ndesc.getError()); - } - - return ndesc; - } - - private static MetadataManager getMetadataManager() { - return MetadataManager.getInstance(KylinConfig.getInstanceFromEnv()); - } - - protected static ResourceStore getStore() { - return ResourceStore.getStore(KylinConfig.getInstanceFromEnv()); - } -}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/cube/src/main/java/org/apache/kylin/cube/CubeDimensionDeriver.java ---------------------------------------------------------------------- diff --git a/cube/src/main/java/org/apache/kylin/cube/CubeDimensionDeriver.java b/cube/src/main/java/org/apache/kylin/cube/CubeDimensionDeriver.java deleted file mode 100644 index 8d771b9..0000000 --- a/cube/src/main/java/org/apache/kylin/cube/CubeDimensionDeriver.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.cube; - -import java.util.Collection; -import java.util.HashSet; - -import org.apache.kylin.metadata.model.TblColRef; - -/** - * Created by Hongbin Ma(Binmahone) on 1/8/15. - * - * the unified logic for defining a sql's dimension - */ -public class CubeDimensionDeriver { - - public static Collection<TblColRef> getDimensionColumns(Collection<TblColRef> groupByColumns, Collection<TblColRef> filterColumns) { - Collection<TblColRef> dimensionColumns = new HashSet<TblColRef>(); - dimensionColumns.addAll(groupByColumns); - dimensionColumns.addAll(filterColumns); - return dimensionColumns; - } -} http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/cube/src/main/java/org/apache/kylin/cube/CubeInstance.java ---------------------------------------------------------------------- diff --git a/cube/src/main/java/org/apache/kylin/cube/CubeInstance.java b/cube/src/main/java/org/apache/kylin/cube/CubeInstance.java deleted file mode 100644 index 7773351..0000000 --- a/cube/src/main/java/org/apache/kylin/cube/CubeInstance.java +++ /dev/null @@ -1,400 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.cube; - -import java.util.ArrayList; -import java.util.List; - -import org.apache.kylin.common.KylinConfig; -import org.apache.kylin.common.persistence.ResourceStore; -import org.apache.kylin.common.persistence.RootPersistentEntity; -import org.apache.kylin.cube.model.CubeDesc; -import org.apache.kylin.metadata.model.MeasureDesc; -import org.apache.kylin.metadata.model.SegmentStatusEnum; -import org.apache.kylin.metadata.model.TblColRef; -import org.apache.kylin.metadata.realization.CapabilityResult; -import org.apache.kylin.metadata.realization.IRealization; -import org.apache.kylin.metadata.realization.RealizationStatusEnum; -import org.apache.kylin.metadata.realization.RealizationType; -import org.apache.kylin.metadata.realization.SQLDigest; - -import com.fasterxml.jackson.annotation.JsonAutoDetect; -import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility; -import com.fasterxml.jackson.annotation.JsonIgnore; -import com.fasterxml.jackson.annotation.JsonManagedReference; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.google.common.base.Objects; -import com.google.common.collect.Lists; - -@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE) -public class CubeInstance extends RootPersistentEntity implements IRealization { - - public static CubeInstance create(String cubeName, String projectName, CubeDesc cubeDesc) { - CubeInstance cubeInstance = new CubeInstance(); - - cubeInstance.setConfig(cubeDesc.getConfig()); - cubeInstance.setName(cubeName); - cubeInstance.setDescName(cubeDesc.getName()); - cubeInstance.setCreateTimeUTC(System.currentTimeMillis()); - cubeInstance.setSegments(new ArrayList<CubeSegment>()); - cubeInstance.setStatus(RealizationStatusEnum.DISABLED); - cubeInstance.updateRandomUuid(); - cubeInstance.setProjectName(projectName); - - return cubeInstance; - } - - @JsonIgnore - private KylinConfig config; - @JsonProperty("name") - private String name; - @JsonProperty("owner") - private String owner; - @JsonProperty("version") - private String version; // user info only, we don't do version control - @JsonProperty("descriptor") - private String descName; - // Mark cube priority for query - @JsonProperty("cost") - private int cost = 50; - @JsonProperty("status") - private RealizationStatusEnum status; - - @JsonManagedReference - @JsonProperty("segments") - private List<CubeSegment> segments = new ArrayList<CubeSegment>(); - - @JsonProperty("create_time_utc") - private long createTimeUTC; - - private String projectName; - - public List<CubeSegment> getBuildingSegments() { - List<CubeSegment> buildingSegments = new ArrayList<CubeSegment>(); - if (null != segments) { - for (CubeSegment segment : segments) { - if (SegmentStatusEnum.NEW == segment.getStatus() || SegmentStatusEnum.READY_PENDING == segment.getStatus()) { - buildingSegments.add(segment); - } - } - } - - return buildingSegments; - } - - public List<CubeSegment> getMergingSegments(CubeSegment mergedSegment) { - List<CubeSegment> mergingSegments = new ArrayList<CubeSegment>(); - if (null != this.segments) { - for (CubeSegment segment : this.segments) { - if (!mergedSegment.equals(segment) // - && mergedSegment.getDateRangeStart() <= segment.getDateRangeStart() && mergedSegment.getDateRangeEnd() >= segment.getDateRangeEnd()) { - mergingSegments.add(segment); - } - } - } - return mergingSegments; - } - - public CubeDesc getDescriptor() { - return CubeDescManager.getInstance(config).getCubeDesc(descName); - } - - public boolean isReady() { - return getStatus() == RealizationStatusEnum.READY; - } - - public String getResourcePath() { - return concatResourcePath(name); - } - - public static String concatResourcePath(String cubeName) { - return ResourceStore.CUBE_RESOURCE_ROOT + "/" + cubeName + ".json"; - } - - @Override - public String toString() { - return getCanonicalName(); - } - - // ============================================================================ - - @JsonProperty("size_kb") - public long getSizeKB() { - long sizeKb = 0L; - - for (CubeSegment cubeSegment : this.getSegments(SegmentStatusEnum.READY)) { - sizeKb += cubeSegment.getSizeKB(); - } - - return sizeKb; - } - - @JsonProperty("input_records_count") - public long getInputRecordCount() { - long sizeRecordCount = 0L; - - for (CubeSegment cubeSegment : this.getSegments(SegmentStatusEnum.READY)) { - sizeRecordCount += cubeSegment.getInputRecords(); - } - - return sizeRecordCount; - } - - @JsonProperty("input_records_size") - public long getInputRecordSize() { - long sizeRecordSize = 0L; - - for (CubeSegment cubeSegment : this.getSegments(SegmentStatusEnum.READY)) { - sizeRecordSize += cubeSegment.getInputRecordsSize(); - } - - return sizeRecordSize; - } - - public KylinConfig getConfig() { - return config; - } - - public void setConfig(KylinConfig config) { - this.config = config; - } - - @Override - public String getName() { - return name; - } - - @Override - public String getCanonicalName() { - return getType() + "[name=" + name + "]"; - } - - @Override - public String getFactTable() { - return this.getDescriptor().getFactTable(); - } - - @Override - public List<MeasureDesc> getMeasures() { - return getDescriptor().getMeasures(); - } - - public void setName(String name) { - this.name = name; - } - - public String getOwner() { - return owner; - } - - public void setOwner(String owner) { - this.owner = owner; - } - - public String getVersion() { - return version; - } - - public void setVersion(String version) { - this.version = version; - } - - public String getDescName() { - return descName.toUpperCase(); - } - - public String getOriginDescName() { - return descName; - } - - public void setDescName(String descName) { - this.descName = descName; - } - - public int getCost() { - return cost; - } - - public void setCost(int cost) { - this.cost = cost; - } - - public RealizationStatusEnum getStatus() { - return status; - } - - public void setStatus(RealizationStatusEnum status) { - this.status = status; - } - - public CubeSegment getFirstSegment() { - if (this.segments == null || this.segments.size() == 0) { - return null; - } else { - return this.segments.get(0); - } - } - - public CubeSegment getLatestReadySegment() { - CubeSegment latest = null; - for (int i = segments.size() - 1; i >= 0; i--) { - CubeSegment seg = segments.get(i); - if (seg.getStatus() != SegmentStatusEnum.READY) - continue; - if (latest == null || latest.getDateRangeEnd() < seg.getDateRangeEnd()) { - latest = seg; - } - } - return latest; - } - - public List<CubeSegment> getSegments() { - return segments; - } - - public List<CubeSegment> getSegments(SegmentStatusEnum status) { - List<CubeSegment> result = new ArrayList<CubeSegment>(); - - for (CubeSegment segment : segments) { - if (segment.getStatus() == status) { - result.add(segment); - } - } - - return result; - } - - public List<CubeSegment> getSegment(SegmentStatusEnum status) { - List<CubeSegment> result = Lists.newArrayList(); - for (CubeSegment segment : segments) { - if (segment.getStatus() == status) { - result.add(segment); - } - } - return result; - } - - public CubeSegment getSegment(String name, SegmentStatusEnum status) { - for (CubeSegment segment : segments) { - if ((null != segment.getName() && segment.getName().equals(name)) && segment.getStatus() == status) { - return segment; - } - } - - return null; - } - - public void setSegments(List<CubeSegment> segments) { - this.segments = segments; - } - - public CubeSegment getSegmentById(String segmentId) { - for (CubeSegment segment : segments) { - if (Objects.equal(segment.getUuid(), segmentId)) { - return segment; - } - } - return null; - } - - public long getCreateTimeUTC() { - return createTimeUTC; - } - - public void setCreateTimeUTC(long createTimeUTC) { - this.createTimeUTC = createTimeUTC; - } - - @Override - public CapabilityResult isCapable(SQLDigest digest) { - CapabilityResult result = CubeCapabilityChecker.check(this, digest); - if (result.capable) { - result.cost = getCost(digest); - for (CapabilityResult.CapabilityInfluence i : result.influences) { - result.cost *= (i.suggestCostMultiplier() == 0) ? 1.0 : i.suggestCostMultiplier(); - } - } else { - result.cost = -1; - } - return result; - } - - private int getCost(SQLDigest digest) { - return cost; - } - - @Override - public RealizationType getType() { - return RealizationType.CUBE; - } - - @Override - public List<TblColRef> getAllColumns() { - return Lists.newArrayList(getDescriptor().listAllColumns()); - } - - public String getProjectName() { - return projectName; - } - - public void setProjectName(String projectName) { - this.projectName = projectName; - } - - - public boolean needAutoMerge() { - if (!this.getDescriptor().getModel().getPartitionDesc().isPartitioned()) - return false; - - return this.getDescriptor().getAutoMergeTimeRanges() != null && this.getDescriptor().getAutoMergeTimeRanges().length > 0; - } - - - @Override - public long getDateRangeStart() { - List<CubeSegment> readySegs = getSegments(SegmentStatusEnum.READY); - - long startTime = Long.MAX_VALUE; - for (CubeSegment seg : readySegs) { - if (seg.getDateRangeStart() < startTime) - startTime = seg.getDateRangeStart(); - } - - return startTime; - } - - @Override - public long getDateRangeEnd() { - - List<CubeSegment> readySegs = getSegments(SegmentStatusEnum.READY); - - long endTime = 0; - for (CubeSegment seg : readySegs) { - if (seg.getDateRangeEnd() > endTime) - endTime = seg.getDateRangeEnd(); - } - - return endTime; - } - - @Override - public List<TblColRef> getAllDimensions() { - return Lists.newArrayList(getDescriptor().listDimensionColumnsIncludingDerived()); - } -} http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/cube/src/main/java/org/apache/kylin/cube/CubeManager.java ---------------------------------------------------------------------- diff --git a/cube/src/main/java/org/apache/kylin/cube/CubeManager.java b/cube/src/main/java/org/apache/kylin/cube/CubeManager.java deleted file mode 100644 index d3bbf59..0000000 --- a/cube/src/main/java/org/apache/kylin/cube/CubeManager.java +++ /dev/null @@ -1,750 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.cube; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.Date; -import java.util.Iterator; -import java.util.List; -import java.util.Random; -import java.util.UUID; -import java.util.concurrent.ConcurrentHashMap; - -import org.apache.commons.lang3.StringUtils; -import org.apache.kylin.common.KylinConfig; -import org.apache.kylin.common.persistence.JsonSerializer; -import org.apache.kylin.common.persistence.ResourceStore; -import org.apache.kylin.common.persistence.Serializer; -import org.apache.kylin.common.restclient.Broadcaster; -import org.apache.kylin.common.restclient.CaseInsensitiveStringCache; -import org.apache.kylin.common.util.Dictionary; -import org.apache.kylin.common.util.Pair; -import org.apache.kylin.cube.cuboid.Cuboid; -import org.apache.kylin.cube.model.CubeDesc; -import org.apache.kylin.cube.model.DimensionDesc; -import org.apache.kylin.dict.DictionaryInfo; -import org.apache.kylin.dict.DictionaryManager; -import org.apache.kylin.dict.lookup.HiveTable; -import org.apache.kylin.dict.lookup.LookupStringTable; -import org.apache.kylin.dict.lookup.SnapshotManager; -import org.apache.kylin.dict.lookup.SnapshotTable; -import org.apache.kylin.metadata.MetadataManager; -import org.apache.kylin.metadata.model.PartitionDesc; -import org.apache.kylin.metadata.model.SegmentStatusEnum; -import org.apache.kylin.metadata.model.TableDesc; -import org.apache.kylin.metadata.model.TblColRef; -import org.apache.kylin.metadata.project.ProjectManager; -import org.apache.kylin.metadata.realization.IRealization; -import org.apache.kylin.metadata.realization.IRealizationConstants; -import org.apache.kylin.metadata.realization.IRealizationProvider; -import org.apache.kylin.metadata.realization.RealizationStatusEnum; -import org.apache.kylin.metadata.realization.RealizationType; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.google.common.collect.HashMultimap; -import com.google.common.collect.Lists; -import com.google.common.collect.Multimap; - -/** - * @author yangli9 - */ -public class CubeManager implements IRealizationProvider { - - private static String ALPHA_NUM = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"; - - private static int HBASE_TABLE_LENGTH = 10; - public static final Serializer<CubeInstance> CUBE_SERIALIZER = new JsonSerializer<CubeInstance>(CubeInstance.class); - - private static final Logger logger = LoggerFactory.getLogger(CubeManager.class); - - // static cached instances - private static final ConcurrentHashMap<KylinConfig, CubeManager> CACHE = new ConcurrentHashMap<KylinConfig, CubeManager>(); - - public static CubeManager getInstance(KylinConfig config) { - CubeManager r = CACHE.get(config); - if (r != null) { - return r; - } - - synchronized (CubeManager.class) { - r = CACHE.get(config); - if (r != null) { - return r; - } - try { - r = new CubeManager(config); - CACHE.put(config, r); - if (CACHE.size() > 1) { - logger.warn("More than one cubemanager singleton exist"); - } - return r; - } catch (IOException e) { - throw new IllegalStateException("Failed to init CubeManager from " + config, e); - } - } - } - - public static void clearCache() { - CACHE.clear(); - } - - // ============================================================================ - - private KylinConfig config; - // cube name ==> CubeInstance - private CaseInsensitiveStringCache<CubeInstance> cubeMap = new CaseInsensitiveStringCache<CubeInstance>(Broadcaster.TYPE.CUBE); - // "table/column" ==> lookup table - // private SingleValueCache<String, LookupStringTable> lookupTables = new SingleValueCache<String, LookupStringTable>(Broadcaster.TYPE.METADATA); - - // for generation hbase table name of a new segment - private Multimap<String, String> usedStorageLocation = HashMultimap.create(); - - private CubeManager(KylinConfig config) throws IOException { - logger.info("Initializing CubeManager with config " + config); - this.config = config; - - loadAllCubeInstance(); - } - - public List<CubeInstance> listAllCubes() { - return new ArrayList<CubeInstance>(cubeMap.values()); - } - - public CubeInstance getCube(String cubeName) { - cubeName = cubeName.toUpperCase(); - return cubeMap.get(cubeName); - } - - /** - * Get related Cubes by cubedesc name. By default, the desc name will be - * translated into upper case. - * - * @param descName CubeDesc name - * @return - */ - public List<CubeInstance> getCubesByDesc(String descName) { - - descName = descName.toUpperCase(); - List<CubeInstance> list = listAllCubes(); - List<CubeInstance> result = new ArrayList<CubeInstance>(); - Iterator<CubeInstance> it = list.iterator(); - while (it.hasNext()) { - CubeInstance ci = it.next(); - if (descName.equals(ci.getDescName())) { - result.add(ci); - } - } - return result; - } - - public DictionaryInfo buildDictionary(CubeSegment cubeSeg, TblColRef col, String factColumnsPath) throws IOException { - CubeDesc cubeDesc = cubeSeg.getCubeDesc(); - if (!cubeDesc.getRowkey().isUseDictionary(col)) - return null; - - DictionaryManager dictMgr = getDictionaryManager(); - DictionaryInfo dictInfo = dictMgr.buildDictionary(cubeDesc.getModel(), cubeDesc.getRowkey().getDictionary(col), col, factColumnsPath); - - if (dictInfo != null) { - cubeSeg.putDictResPath(col, dictInfo.getResourcePath()); - saveResource(cubeSeg.getCubeInstance()); - } - - return dictInfo; - } - - /** - * return null if no dictionary for given column - */ - public Dictionary<String> getDictionary(CubeSegment cubeSeg, TblColRef col) { - DictionaryInfo info = null; - try { - DictionaryManager dictMgr = getDictionaryManager(); - // logger.info("Using metadata url " + metadataUrl + - // " for DictionaryManager"); - String dictResPath = cubeSeg.getDictResPath(col); - if (dictResPath == null) - return null; - - info = dictMgr.getDictionaryInfo(dictResPath); - if (info == null) - throw new IllegalStateException("No dictionary found by " + dictResPath + ", invalid cube state; cube segment" + cubeSeg + ", col " + col); - } catch (IOException e) { - throw new IllegalStateException("Failed to get dictionary for cube segment" + cubeSeg + ", col" + col, e); - } - - return (Dictionary<String>) info.getDictionaryObject(); - } - - public SnapshotTable buildSnapshotTable(CubeSegment cubeSeg, String lookupTable) throws IOException { - MetadataManager metaMgr = getMetadataManager(); - SnapshotManager snapshotMgr = getSnapshotManager(); - - HiveTable hiveTable = new HiveTable(metaMgr, lookupTable); - TableDesc tableDesc = metaMgr.getTableDesc(lookupTable); - SnapshotTable snapshot = snapshotMgr.buildSnapshot(hiveTable, tableDesc); - - cubeSeg.putSnapshotResPath(lookupTable, snapshot.getResourcePath()); - - saveResource(cubeSeg.getCubeInstance()); - - return snapshot; - } - - // sync on update - public CubeInstance dropCube(String cubeName, boolean deleteDesc) throws IOException { - logger.info("Dropping cube '" + cubeName + "'"); - // load projects before remove cube from project - - ResourceStore store = getStore(); - - // delete cube instance and cube desc - CubeInstance cube = getCube(cubeName); - - if (deleteDesc && cube.getDescriptor() != null) - store.deleteResource(cube.getDescriptor().getResourcePath()); - - store.deleteResource(cube.getResourcePath()); - - // delete cube from project - ProjectManager.getInstance(config).removeRealizationsFromProjects(RealizationType.CUBE, cubeName); - - // clean cube cache - this.afterCubeDropped(cube); - - return cube; - } - - // sync on update - public CubeInstance createCube(String cubeName, String projectName, CubeDesc desc, String owner) throws IOException { - logger.info("Creating cube '" + projectName + "-->" + cubeName + "' from desc '" + desc.getName() + "'"); - - // save cube resource - CubeInstance cube = CubeInstance.create(cubeName, projectName, desc); - cube.setOwner(owner); - saveResource(cube); - - ProjectManager.getInstance(config).moveRealizationToProject(RealizationType.CUBE, cubeName, projectName, owner); - - return cube; - } - - public CubeInstance updateCube(CubeInstance cube) throws IOException { - logger.info("Updating cube instance '" + cube.getName()); - saveResource(cube); - return cube; - } - - public Pair<CubeSegment, CubeSegment> appendAndMergeSegments(CubeInstance cube, long endDate) throws IOException { - checkNoBuildingSegment(cube); - checkCubeIsPartitioned(cube); - - if (cube.getSegments().size() == 0) - throw new IllegalStateException("expect at least one existing segment"); - - long appendStart = calculateStartDateForAppendSegment(cube); - CubeSegment appendSegment = newSegment(cube, appendStart, endDate); - - long startDate = cube.getDescriptor().getModel().getPartitionDesc().getPartitionDateStart(); - CubeSegment mergeSegment = newSegment(cube, startDate, endDate); - - validateNewSegments(cube, mergeSegment); - cube.getSegments().add(appendSegment); - cube.getSegments().add(mergeSegment); - Collections.sort(cube.getSegments()); - updateCube(cube); - - return new Pair<CubeSegment, CubeSegment>(appendSegment, mergeSegment); - } - - public CubeSegment appendSegments(CubeInstance cube, long endDate) throws IOException { - checkNoBuildingSegment(cube); - - CubeSegment newSegment; - if (cube.getDescriptor().getModel().getPartitionDesc().isPartitioned()) { - long startDate = calculateStartDateForAppendSegment(cube); - newSegment = newSegment(cube, startDate, endDate); - } else { - newSegment = newSegment(cube, 0, Long.MAX_VALUE); - } - - validateNewSegments(cube, newSegment); - cube.getSegments().add(newSegment); - Collections.sort(cube.getSegments()); - updateCube(cube); - - return newSegment; - } - - public CubeSegment refreshSegment(CubeInstance cube, long startDate, long endDate) throws IOException { - checkNoBuildingSegment(cube); - - CubeSegment newSegment = newSegment(cube, startDate, endDate); - cube.getSegments().add(newSegment); - Collections.sort(cube.getSegments()); - updateCube(cube); - - return newSegment; - } - - public CubeSegment mergeSegments(CubeInstance cube, final long startDate, final long endDate, boolean forceMergeEmptySeg) throws IOException { - checkNoBuildingSegment(cube); - checkCubeIsPartitioned(cube); - - Pair<Long, Long> range = alignMergeRange(cube, startDate, endDate); - CubeSegment newSegment = newSegment(cube, range.getFirst(), range.getSecond()); - List<CubeSegment> mergingSegments = cube.getMergingSegments(newSegment); - - if (forceMergeEmptySeg == false) { - List<String> emptySegment = Lists.newArrayList(); - for (CubeSegment seg : mergingSegments) { - if (seg.getSizeKB() == 0) { - emptySegment.add(seg.getName()); - } - } - if (emptySegment.size() > 0) { - throw new IllegalArgumentException("Empty cube segment found, couldn't merge unless 'forceMergeEmptySegment' set to true: " + emptySegment); - } - } - - validateNewSegments(cube, newSegment); - cube.getSegments().add(newSegment); - Collections.sort(cube.getSegments()); - updateCube(cube); - - return newSegment; - } - - private Pair<Long, Long> alignMergeRange(CubeInstance cube, long startDate, long endDate) { - List<CubeSegment> readySegments = cube.getSegment(SegmentStatusEnum.READY); - if (readySegments.isEmpty()) { - throw new IllegalStateException("there are no segments in ready state"); - } - long start = Long.MAX_VALUE; - long end = Long.MIN_VALUE; - for (CubeSegment readySegment : readySegments) { - if (hasOverlap(startDate, endDate, readySegment.getDateRangeStart(), readySegment.getDateRangeEnd())) { - if (start > readySegment.getDateRangeStart()) { - start = readySegment.getDateRangeStart(); - } - if (end < readySegment.getDateRangeEnd()) { - end = readySegment.getDateRangeEnd(); - } - } - } - return new Pair<Long, Long>(start, end); - } - - private boolean hasOverlap(long startDate, long endDate, long anotherStartDate, long anotherEndDate) { - if (startDate >= endDate) { - throw new IllegalArgumentException("startDate must be less than endDate"); - } - if (anotherStartDate >= anotherEndDate) { - throw new IllegalArgumentException("anotherStartDate must be less than anotherEndDate"); - } - if (startDate <= anotherStartDate && anotherStartDate < endDate) { - return true; - } - if (startDate < anotherEndDate && anotherEndDate <= endDate) { - return true; - } - return false; - } - - private long calculateStartDateForAppendSegment(CubeInstance cube) { - List<CubeSegment> existing = cube.getSegments(); - if (existing.isEmpty()) { - return cube.getDescriptor().getModel().getPartitionDesc().getPartitionDateStart(); - } else { - return existing.get(existing.size() - 1).getDateRangeEnd(); - } - } - - private void checkNoBuildingSegment(CubeInstance cube) { - if (cube.getBuildingSegments().size() > 0) { - throw new IllegalStateException("There is already a building segment!"); - } - } - - private void checkCubeIsPartitioned(CubeInstance cube) { - if (cube.getDescriptor().getModel().getPartitionDesc().isPartitioned() == false) { - throw new IllegalStateException("there is no partition date column specified, only full build is supported"); - } - } - - public void updateSegmentOnJobDiscard(CubeInstance cubeInstance, String segmentName) throws IOException { - for (int i = 0; i < cubeInstance.getSegments().size(); i++) { - CubeSegment segment = cubeInstance.getSegments().get(i); - if (segment.getName().equals(segmentName) && segment.getStatus() != SegmentStatusEnum.READY) { - cubeInstance.getSegments().remove(segment); - } - } - updateCube(cubeInstance); - } - - /** - * After cube update, reload cube related cache - * - * @param cubeName - */ - public void loadCubeCache(String cubeName) { - try { - loadCubeInstance(CubeInstance.concatResourcePath(cubeName)); - } catch (IOException e) { - logger.error(e.getLocalizedMessage(), e); - } - } - - /** - * After cube deletion, remove cube related cache - * - * @param cube - */ - public void removeCubeCache(CubeInstance cube) { - final String cubeName = cube.getName().toUpperCase(); - cubeMap.remove(cubeName); - usedStorageLocation.removeAll(cubeName); - Cuboid.reloadCache(cube.getDescName()); - } - - public void removeCubeCacheLocal(String cubeName) { - cubeMap.removeLocal(cubeName); - usedStorageLocation.removeAll(cubeName); - } - - public LookupStringTable getLookupTable(CubeSegment cubeSegment, DimensionDesc dim) { - - String tableName = dim.getTable(); - String[] pkCols = dim.getJoin().getPrimaryKey(); - String snapshotResPath = cubeSegment.getSnapshotResPath(tableName); - if (snapshotResPath == null) - throw new IllegalStateException("No snaphot for table '" + tableName + "' found on cube segment" + cubeSegment.getCubeInstance().getName() + "/" + cubeSegment); - - try { - SnapshotTable snapshot = getSnapshotManager().getSnapshotTable(snapshotResPath); - TableDesc tableDesc = getMetadataManager().getTableDesc(tableName); - return new LookupStringTable(tableDesc, pkCols, snapshot); - } catch (IOException e) { - throw new IllegalStateException("Failed to load lookup table " + tableName + " from snapshot " + snapshotResPath, e); - } - } - - private void saveResource(CubeInstance cube) throws IOException { - ResourceStore store = getStore(); - store.putResource(cube.getResourcePath(), cube, CUBE_SERIALIZER); - this.afterCubeUpdated(cube); - } - - private void afterCubeUpdated(CubeInstance updatedCube) { - cubeMap.put(updatedCube.getName(), updatedCube); - } - - private void afterCubeDropped(CubeInstance droppedCube) { - removeCubeCache(droppedCube); - } - - private CubeSegment newSegment(CubeInstance cubeInstance, long startDate, long endDate) { - if (startDate >= endDate) - throw new IllegalArgumentException("New segment range invalid, start date must be earlier than end date, " + startDate + " < " + endDate); - - CubeSegment segment = new CubeSegment(); - String incrementalSegName = CubeSegment.getSegmentName(startDate, endDate); - segment.setUuid(UUID.randomUUID().toString()); - segment.setName(incrementalSegName); - Date creatTime = new Date(); - segment.setCreateTimeUTC(creatTime.getTime()); - segment.setDateRangeStart(startDate); - segment.setDateRangeEnd(endDate); - segment.setStatus(SegmentStatusEnum.NEW); - segment.setStorageLocationIdentifier(generateStorageLocation()); - - segment.setCubeInstance(cubeInstance); - - segment.validate(); - return segment; - } - - private String generateStorageLocation() { - String namePrefix = IRealizationConstants.CubeHbaseStorageLocationPrefix; - String tableName = ""; - do { - StringBuffer sb = new StringBuffer(); - sb.append(namePrefix); - Random ran = new Random(); - for (int i = 0; i < HBASE_TABLE_LENGTH; i++) { - sb.append(ALPHA_NUM.charAt(ran.nextInt(ALPHA_NUM.length()))); - } - tableName = sb.toString(); - } while (this.usedStorageLocation.containsValue(tableName)); - - return tableName; - } - - public void promoteNewlyBuiltSegments(CubeInstance cube, CubeSegment... newSegments) throws IOException { - List<CubeSegment> tobe = calculateToBeSegments(cube); - - for (CubeSegment seg : newSegments) { - if (tobe.contains(seg) == false) - throw new IllegalStateException("For cube " + cube + ", segment " + seg + " is expected but not in the tobe " + tobe); - - if (StringUtils.isBlank(seg.getStorageLocationIdentifier())) - throw new IllegalStateException("For cube " + cube + ", segment " + seg + " missing StorageLocationIdentifier"); - - if (StringUtils.isBlank(seg.getLastBuildJobID())) - throw new IllegalStateException("For cube " + cube + ", segment " + seg + " missing LastBuildJobID"); - - seg.setStatus(SegmentStatusEnum.READY); - } - - for (CubeSegment seg : tobe) { - if (isReady(seg) == false) - throw new IllegalStateException("For cube " + cube + ", segment " + seg + " should be READY but is not"); - } - - cube.setSegments(tobe); - cube.setStatus(RealizationStatusEnum.READY); - - logger.info("Promoting cube " + cube + ", new segments " + newSegments); - saveResource(cube); - } - - private void validateNewSegments(CubeInstance cube, CubeSegment... newSegments) { - List<CubeSegment> tobe = calculateToBeSegments(cube, newSegments); - List<CubeSegment> newList = Arrays.asList(newSegments); - if (tobe.containsAll(newList) == false) { - throw new IllegalStateException("For cube " + cube + ", the new segments " + newList + " do not fit in its current " + cube.getSegments() + "; the resulted tobe is " + tobe); - } - } - - /** - * Smartly figure out the TOBE segments once all new segments are built. - * - Ensures no gap, no overlap - * - Favors new segments over the old - * - Favors big segments over the small - */ - private List<CubeSegment> calculateToBeSegments(CubeInstance cube, CubeSegment... newSegments) { - CubeDesc cubeDesc = cube.getDescriptor(); - PartitionDesc partDesc = cubeDesc.getModel().getPartitionDesc(); - - List<CubeSegment> tobe = Lists.newArrayList(cube.getSegments()); - if (newSegments != null) - tobe.addAll(Arrays.asList(newSegments)); - if (tobe.size() == 0) - return tobe; - - // sort by start time, then end time - Collections.sort(tobe); - - // check first segment start time - CubeSegment firstSeg = tobe.get(0); - firstSeg.validate(); - - for (int i = 0, j = 1; j < tobe.size();) { - CubeSegment is = tobe.get(i); - CubeSegment js = tobe.get(j); - js.validate(); - - // check i is either ready or new - if (!isNew(is) && !isReady(is)) { - tobe.remove(i); - continue; - } - - // check j is either ready or new - if (!isNew(js) && !isReady(js)) { - tobe.remove(j); - continue; - } - - // if i, j competes - if (is.getDateRangeStart() == js.getDateRangeStart()) { - // if both new or ready, favor the bigger segment - if (isReady(is) && isReady(js) || isNew(is) && isNew(js)) { - if (is.getDateRangeEnd() <= js.getDateRangeEnd()) { - tobe.remove(i); - } else { - tobe.remove(j); - } - } - // otherwise, favor the new segment - else if (isNew(is)) { - tobe.remove(j); - } else { - tobe.remove(i); - } - continue; - } - - // if i, j in sequence - if (is.getDateRangeEnd() == js.getDateRangeStart()) { - i++; - j++; - continue; - } - - // seems j not fitting - tobe.remove(j); - } - - return tobe; - } - - private boolean isReady(CubeSegment seg) { - return seg.getStatus() == SegmentStatusEnum.READY; - } - - private boolean isNew(CubeSegment seg) { - return seg.getStatus() == SegmentStatusEnum.NEW || seg.getStatus() == SegmentStatusEnum.READY_PENDING; - } - - private void loadAllCubeInstance() throws IOException { - ResourceStore store = getStore(); - List<String> paths = store.collectResourceRecursively(ResourceStore.CUBE_RESOURCE_ROOT, ".json"); - - logger.debug("Loading Cube from folder " + store.getReadableResourcePath(ResourceStore.CUBE_RESOURCE_ROOT)); - - for (String path : paths) { - loadCubeInstance(path); - } - - logger.debug("Loaded " + paths.size() + " Cube(s)"); - } - - private synchronized CubeInstance loadCubeInstance(String path) throws IOException { - ResourceStore store = getStore(); - - CubeInstance cubeInstance; - try { - cubeInstance = store.getResource(path, CubeInstance.class, CUBE_SERIALIZER); - cubeInstance.setConfig(config); - - if (StringUtils.isBlank(cubeInstance.getName())) - throw new IllegalStateException("CubeInstance name must not be blank, at " + path); - - if (cubeInstance.getDescriptor() == null) - throw new IllegalStateException("CubeInstance desc not found '" + cubeInstance.getDescName() + "', at " + path); - - final String cubeName = cubeInstance.getName(); - cubeMap.putLocal(cubeName, cubeInstance); - - for (CubeSegment segment : cubeInstance.getSegments()) { - usedStorageLocation.put(cubeName, segment.getStorageLocationIdentifier()); - } - - return cubeInstance; - } catch (Exception e) { - logger.error("Error during load cube instance " + path, e); - return null; - } - } - - public CubeSegment autoMergeCubeSegments(CubeInstance cube) throws IOException { - if (!cube.needAutoMerge()) { - logger.debug("Cube " + cube.getName() + " doesn't need auto merge"); - return null; - } - - if (cube.getBuildingSegments().size() > 0) { - logger.debug("Cube " + cube.getName() + " has bulding segment, will not trigger merge at this moment"); - return null; - } - - List<CubeSegment> readySegments = Lists.newArrayList(cube.getSegment(SegmentStatusEnum.READY)); - - if (readySegments.size() == 0) { - logger.debug("Cube " + cube.getName() + " has no ready segment to merge"); - return null; - } - - long[] timeRanges = cube.getDescriptor().getAutoMergeTimeRanges(); - Arrays.sort(timeRanges); - - CubeSegment newSeg = null; - for (int i = timeRanges.length - 1; i >= 0; i--) { - long toMergeRange = timeRanges[i]; - long currentRange = 0; - long lastEndTime = 0; - List<CubeSegment> toMergeSegments = Lists.newArrayList(); - for (CubeSegment segment : readySegments) { - long thisSegmentRange = segment.getDateRangeEnd() - segment.getDateRangeStart(); - - if (thisSegmentRange >= toMergeRange) { - // this segment and its previous segments will not be merged - toMergeSegments.clear(); - currentRange = 0; - lastEndTime = segment.getDateRangeEnd(); - continue; - } - - if (segment.getDateRangeStart() != lastEndTime && toMergeSegments.isEmpty() == false) { - // gap exists, give up the small segments before the gap; - toMergeSegments.clear(); - currentRange = 0; - } - - currentRange += thisSegmentRange; - if (currentRange < toMergeRange) { - toMergeSegments.add(segment); - lastEndTime = segment.getDateRangeEnd(); - } else { - // merge - toMergeSegments.add(segment); - - newSeg = newSegment(cube, toMergeSegments.get(0).getDateRangeStart(), segment.getDateRangeEnd()); - // only one merge job be created here - return newSeg; - } - } - - } - - return null; - } - - private MetadataManager getMetadataManager() { - return MetadataManager.getInstance(config); - } - - private DictionaryManager getDictionaryManager() { - return DictionaryManager.getInstance(config); - } - - private SnapshotManager getSnapshotManager() { - return SnapshotManager.getInstance(config); - } - - private ResourceStore getStore() { - return ResourceStore.getStore(this.config); - } - - @Override - public RealizationType getRealizationType() { - return RealizationType.CUBE; - } - - @Override - public IRealization getRealization(String name) { - return getCube(name); - } - -} http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/cube/src/main/java/org/apache/kylin/cube/CubeSegment.java ---------------------------------------------------------------------- diff --git a/cube/src/main/java/org/apache/kylin/cube/CubeSegment.java b/cube/src/main/java/org/apache/kylin/cube/CubeSegment.java deleted file mode 100644 index 12f0bf7..0000000 --- a/cube/src/main/java/org/apache/kylin/cube/CubeSegment.java +++ /dev/null @@ -1,348 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.cube; - -import java.text.SimpleDateFormat; -import java.util.Collection; -import java.util.Map; -import java.util.TimeZone; -import java.util.concurrent.ConcurrentHashMap; - -import com.google.common.collect.Maps; -import org.apache.kylin.cube.model.CubeDesc; -import org.apache.kylin.common.util.Dictionary; -import org.apache.kylin.dict.ISegment; -import org.apache.kylin.metadata.model.SegmentStatusEnum; -import org.apache.kylin.metadata.model.TblColRef; - -import com.fasterxml.jackson.annotation.JsonAutoDetect; -import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility; -import com.fasterxml.jackson.annotation.JsonBackReference; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.google.common.base.Objects; - -@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE) -public class CubeSegment implements Comparable<CubeSegment>, ISegment { - - @JsonBackReference - private CubeInstance cubeInstance; - @JsonProperty("uuid") - private String uuid; - @JsonProperty("name") - private String name; - @JsonProperty("storage_location_identifier") - private String storageLocationIdentifier; // HTable name - @JsonProperty("date_range_start") - private long dateRangeStart; - @JsonProperty("date_range_end") - private long dateRangeEnd; - @JsonProperty("status") - private SegmentStatusEnum status; - @JsonProperty("size_kb") - private long sizeKB; - @JsonProperty("input_records") - private long inputRecords; - @JsonProperty("input_records_size") - private long inputRecordsSize; - @JsonProperty("last_build_time") - private long lastBuildTime; - @JsonProperty("last_build_job_id") - private String lastBuildJobID; - @JsonProperty("create_time_utc") - private long createTimeUTC; - - @JsonProperty("binary_signature") - private String binarySignature; // a hash of cube schema and dictionary ID, used for sanity check - - @JsonProperty("dictionaries") - private ConcurrentHashMap<String, String> dictionaries; // table/column ==> dictionary resource path - @JsonProperty("snapshots") - private ConcurrentHashMap<String, String> snapshots; // table name ==> snapshot resource path - - public CubeDesc getCubeDesc() { - return getCubeInstance().getDescriptor(); - } - - /** - * @param startDate - * @param endDate - * @return if(startDate == 0 && endDate == 0), returns "FULL_BUILD", else - * returns "yyyyMMddHHmmss_yyyyMMddHHmmss" - */ - public static String getSegmentName(long startDate, long endDate) { - if (startDate == 0 && endDate == 0) { - return "FULL_BUILD"; - } - - SimpleDateFormat dateFormat = new SimpleDateFormat("yyyyMMddHHmmss"); - dateFormat.setTimeZone(TimeZone.getTimeZone("GMT")); - - return dateFormat.format(startDate) + "_" + dateFormat.format(endDate); - } - - // ============================================================================ - - public String getUuid() { - return uuid; - } - - public void setUuid(String id) { - this.uuid = id; - } - - public String getName() { - return name; - } - - public void setName(String name) { - this.name = name; - } - - public long getDateRangeStart() { - return dateRangeStart; - } - - public void setDateRangeStart(long dateRangeStart) { - this.dateRangeStart = dateRangeStart; - } - - public long getDateRangeEnd() { - return dateRangeEnd; - } - - public void setDateRangeEnd(long dateRangeEnd) { - this.dateRangeEnd = dateRangeEnd; - } - - public SegmentStatusEnum getStatus() { - return status; - } - - public void setStatus(SegmentStatusEnum status) { - this.status = status; - } - - public long getSizeKB() { - return sizeKB; - } - - public void setSizeKB(long sizeKB) { - this.sizeKB = sizeKB; - } - - public long getInputRecords() { - return inputRecords; - } - - public void setInputRecords(long inputRecords) { - this.inputRecords = inputRecords; - } - - public long getInputRecordsSize() { - return inputRecordsSize; - } - - public void setInputRecordsSize(long inputRecordsSize) { - this.inputRecordsSize = inputRecordsSize; - } - - public long getLastBuildTime() { - return lastBuildTime; - } - - public void setLastBuildTime(long lastBuildTime) { - this.lastBuildTime = lastBuildTime; - } - - public String getLastBuildJobID() { - return lastBuildJobID; - } - - public void setLastBuildJobID(String lastBuildJobID) { - this.lastBuildJobID = lastBuildJobID; - } - - public long getCreateTimeUTC() { - return createTimeUTC; - } - - public void setCreateTimeUTC(long createTimeUTC) { - this.createTimeUTC = createTimeUTC; - } - - public String getBinarySignature() { - return binarySignature; - } - - public void setBinarySignature(String binarySignature) { - this.binarySignature = binarySignature; - } - - public CubeInstance getCubeInstance() { - return cubeInstance; - } - - public void setCubeInstance(CubeInstance cubeInstance) { - this.cubeInstance = cubeInstance; - } - - public String getStorageLocationIdentifier() { - - return storageLocationIdentifier; - } - - public Map<String, String> getDictionaries() { - if (dictionaries == null) - dictionaries = new ConcurrentHashMap<String, String>(); - return dictionaries; - } - - public Map<String, String> getSnapshots() { - if (snapshots == null) - snapshots = new ConcurrentHashMap<String, String>(); - return snapshots; - } - - public String getSnapshotResPath(String table) { - return getSnapshots().get(table); - } - - public void putSnapshotResPath(String table, String snapshotResPath) { - getSnapshots().put(table, snapshotResPath); - } - - public Collection<String> getDictionaryPaths() { - return getDictionaries().values(); - } - - public Collection<String> getSnapshotPaths() { - return getSnapshots().values(); - } - - public String getDictResPath(TblColRef col) { - return getDictionaries().get(dictKey(col)); - } - - public void putDictResPath(TblColRef col, String dictResPath) { - getDictionaries().put(dictKey(col), dictResPath); - } - - private String dictKey(TblColRef col) { - return col.getTable() + "/" + col.getName(); - } - - public void setStorageLocationIdentifier(String storageLocationIdentifier) { - this.storageLocationIdentifier = storageLocationIdentifier; - } - - @Override - public int getColumnLength(TblColRef col) { - Dictionary<?> dict = getDictionary(col); - if (dict == null) { - return this.getCubeDesc().getRowkey().getColumnLength(col); - } else { - return dict.getSizeOfId(); - } - } - - @Override - public Dictionary<String> getDictionary(TblColRef col) { - return CubeManager.getInstance(this.getCubeInstance().getConfig()).getDictionary(this, col); - } - - public Map<TblColRef, Dictionary<String>> buildDictionaryMap() { - Map<TblColRef, Dictionary<String>> result = Maps.newHashMap(); - for (TblColRef col : getCubeDesc().getAllColumnsNeedDictionary()) { - result.put(col, (Dictionary<String>) getDictionary(col)); - } - return result; - } - - - public void validate() { - if (cubeInstance.getDescriptor().getModel().getPartitionDesc().isPartitioned() && dateRangeStart >= dateRangeEnd) - throw new IllegalStateException("dateRangeStart(" + dateRangeStart + ") must be greater than dateRangeEnd(" + dateRangeEnd + ") in segment " + this); - } - - @Override - public int compareTo(CubeSegment other) { - long comp = this.dateRangeStart - other.dateRangeStart; - if (comp != 0) - return comp < 0 ? -1 : 1; - - comp = this.dateRangeEnd - other.dateRangeEnd; - if (comp != 0) - return comp < 0 ? -1 : 1; - else - return 0; - } - - @Override - public int hashCode() { - final int prime = 31; - int result = 1; - result = prime * result + ((cubeInstance == null) ? 0 : cubeInstance.hashCode()); - result = prime * result + ((name == null) ? 0 : name.hashCode()); - result = prime * result + ((status == null) ? 0 : status.hashCode()); - return result; - } - - @Override - public boolean equals(Object obj) { - if (this == obj) - return true; - if (obj == null) - return false; - if (getClass() != obj.getClass()) - return false; - CubeSegment other = (CubeSegment) obj; - if (cubeInstance == null) { - if (other.cubeInstance != null) - return false; - } else if (!cubeInstance.equals(other.cubeInstance)) - return false; - if (uuid == null) { - if (other.uuid != null) - return false; - } else if (!uuid.equals(other.uuid)) - return false; - if (name == null) { - if (other.name != null) - return false; - } else if (!name.equals(other.name)) - return false; - if (status != other.status) - return false; - return true; - } - - @Override - public String toString() { - return Objects.toStringHelper(this).add("uuid", uuid).add("create_time_utc:", createTimeUTC).add("name", name).add("last_build_job_id", lastBuildJobID).add("status", status).toString(); - } - - public void setDictionaries(ConcurrentHashMap<String, String> dictionaries) { - this.dictionaries = dictionaries; - } - - public void setSnapshots(ConcurrentHashMap<String, String> snapshots) { - this.snapshots = snapshots; - } - -} http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java ---------------------------------------------------------------------- diff --git a/cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java b/cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java deleted file mode 100644 index 3cdaa93..0000000 --- a/cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.cube.cli; - -import java.io.IOException; - -import org.apache.kylin.common.KylinConfig; -import org.apache.kylin.cube.CubeInstance; -import org.apache.kylin.cube.CubeManager; -import org.apache.kylin.cube.CubeSegment; -import org.apache.kylin.cube.model.DimensionDesc; -import org.apache.kylin.metadata.model.SegmentStatusEnum; -import org.apache.kylin.metadata.model.TblColRef; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class DictionaryGeneratorCLI { - - private static final Logger logger = LoggerFactory.getLogger(DictionaryGeneratorCLI.class); - - public static void processSegment(KylinConfig config, String cubeName, String segmentName, String factColumnsPath) throws IOException { - CubeInstance cube = CubeManager.getInstance(config).getCube(cubeName); - CubeSegment segment = cube.getSegment(segmentName, SegmentStatusEnum.NEW); - - processSegment(config, segment, factColumnsPath); - } - - private static void processSegment(KylinConfig config, CubeSegment cubeSeg, String factColumnsPath) throws IOException { - CubeManager cubeMgr = CubeManager.getInstance(config); - - for (DimensionDesc dim : cubeSeg.getCubeDesc().getDimensions()) { - // dictionary - for (TblColRef col : dim.getColumnRefs()) { - if (cubeSeg.getCubeDesc().getRowkey().isUseDictionary(col)) { - logger.info("Building dictionary for " + col); - cubeMgr.buildDictionary(cubeSeg, col, factColumnsPath); - } - } - - // build snapshot - if (dim.getTable() != null && !dim.getTable().equalsIgnoreCase(cubeSeg.getCubeDesc().getFactTable())) { - // CubeSegment seg = cube.getTheOnlySegment(); - logger.info("Building snapshot of " + dim.getTable()); - cubeMgr.buildSnapshotTable(cubeSeg, dim.getTable()); - logger.info("Checking snapshot of " + dim.getTable()); - cubeMgr.getLookupTable(cubeSeg, dim); // load the table for - // sanity check - } - } - } - -} http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/cube/src/main/java/org/apache/kylin/cube/cli/DumpDictionaryCLI.java ---------------------------------------------------------------------- diff --git a/cube/src/main/java/org/apache/kylin/cube/cli/DumpDictionaryCLI.java b/cube/src/main/java/org/apache/kylin/cube/cli/DumpDictionaryCLI.java deleted file mode 100644 index d4613e9..0000000 --- a/cube/src/main/java/org/apache/kylin/cube/cli/DumpDictionaryCLI.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.cube.cli; - -import java.io.DataInputStream; -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.util.Date; - -import org.apache.kylin.common.util.JsonUtil; -import org.apache.kylin.dict.DictionaryInfo; -import org.apache.kylin.dict.DictionaryInfoSerializer; - -public class DumpDictionaryCLI { - - public static void main(String[] args) throws IOException { - for (String path : args) { - dump(new File(path)); - } - } - - public static void dump(File f) throws IOException { - if (f.isDirectory()) { - for (File c : f.listFiles()) - dump(c); - return; - } - - if (f.getName().endsWith(".dict")) { - DictionaryInfoSerializer ser = new DictionaryInfoSerializer(); - DictionaryInfo dictInfo = ser.deserialize(new DataInputStream(new FileInputStream(f))); - - System.out.println("============================================================================"); - System.out.println("File: " + f.getAbsolutePath()); - System.out.println(new Date(dictInfo.getLastModified())); - System.out.println(JsonUtil.writeValueAsIndentString(dictInfo)); - dictInfo.getDictionaryObject().dump(System.out); - System.out.println(); - } - } -} http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/cube/src/main/java/org/apache/kylin/cube/common/RowKeySplitter.java ---------------------------------------------------------------------- diff --git a/cube/src/main/java/org/apache/kylin/cube/common/RowKeySplitter.java b/cube/src/main/java/org/apache/kylin/cube/common/RowKeySplitter.java deleted file mode 100644 index 4821c94..0000000 --- a/cube/src/main/java/org/apache/kylin/cube/common/RowKeySplitter.java +++ /dev/null @@ -1,91 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.cube.common; - -import org.apache.kylin.common.util.Bytes; -import org.apache.kylin.common.util.SplittedBytes; -import org.apache.kylin.cube.CubeSegment; -import org.apache.kylin.cube.cuboid.Cuboid; -import org.apache.kylin.cube.kv.RowConstants; -import org.apache.kylin.cube.kv.RowKeyColumnIO; -import org.apache.kylin.cube.model.CubeDesc; -import org.apache.kylin.metadata.model.TblColRef; - -/** - * @author George Song (ysong1) - * - */ -public class RowKeySplitter { - - private CubeDesc cubeDesc; - private RowKeyColumnIO colIO; - - private SplittedBytes[] splitBuffers; - private int bufferSize; - - public SplittedBytes[] getSplitBuffers() { - return splitBuffers; - } - - public int getBufferSize() { - return bufferSize; - } - - public RowKeySplitter(CubeSegment cubeSeg, int splitLen, int bytesLen) { - this.cubeDesc = cubeSeg.getCubeDesc(); - this.colIO = new RowKeyColumnIO(cubeSeg); - - this.splitBuffers = new SplittedBytes[splitLen]; - for (int i = 0; i < splitLen; i++) { - this.splitBuffers[i] = new SplittedBytes(bytesLen); - } - this.bufferSize = 0; - } - - /** - * @param bytes - * @param byteLen - * @return cuboid ID - */ - public long split(byte[] bytes, int byteLen) { - this.bufferSize = 0; - int offset = 0; - - // extract cuboid id - SplittedBytes cuboidIdSplit = this.splitBuffers[this.bufferSize++]; - cuboidIdSplit.length = RowConstants.ROWKEY_CUBOIDID_LEN; - System.arraycopy(bytes, offset, cuboidIdSplit.value, 0, RowConstants.ROWKEY_CUBOIDID_LEN); - offset += RowConstants.ROWKEY_CUBOIDID_LEN; - - long cuboidId = Bytes.toLong(cuboidIdSplit.value, 0, cuboidIdSplit.length); - Cuboid cuboid = Cuboid.findById(cubeDesc, cuboidId); - - // rowkey columns - for (int i = 0; i < cuboid.getColumns().size(); i++) { - TblColRef col = cuboid.getColumns().get(i); - int colLength = colIO.getColumnLength(col); - SplittedBytes split = this.splitBuffers[this.bufferSize++]; - split.length = colLength; - System.arraycopy(bytes, offset, split.value, 0, colLength); - offset += colLength; - } - - return cuboidId; - } -}
