minior, refine TableExtDesc,JoinedFlatTable,CubeJoinedFlatTableDesc
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/0762c4dd Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/0762c4dd Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/0762c4dd Branch: refs/heads/master-hadoop3.0 Commit: 0762c4ddbc27be4f471882394e580d94bc836fb5 Parents: a862603 Author: Cheng Wang <[email protected]> Authored: Thu Apr 20 15:09:33 2017 +0800 Committer: liyang-gmt8 <[email protected]> Committed: Tue Apr 25 16:13:53 2017 +0800 ---------------------------------------------------------------------- .../cube/model/CubeJoinedFlatTableDesc.java | 2 +- .../org/apache/kylin/job/JoinedFlatTable.java | 8 +++- .../kylin/metadata/model/TableExtDesc.java | 44 +++++++++++++++++--- 3 files changed, 46 insertions(+), 8 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/0762c4dd/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableDesc.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableDesc.java index f607466..c49d37a 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableDesc.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableDesc.java @@ -44,7 +44,7 @@ public class CubeJoinedFlatTableDesc implements IJoinedFlatTableDesc, java.io.Se private int columnCount = 0; private List<TblColRef> columnList = Lists.newArrayList(); - private Map<TblColRef, Integer> columnIndexMap = Maps.newHashMap();; + private Map<TblColRef, Integer> columnIndexMap = Maps.newHashMap(); public CubeJoinedFlatTableDesc(CubeDesc cubeDesc) { this(cubeDesc, null); http://git-wip-us.apache.org/repos/asf/kylin/blob/0762c4dd/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java ---------------------------------------------------------------------- diff --git a/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java index 54c1e61..a9da822 100644 --- a/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java +++ b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java @@ -107,7 +107,13 @@ public class JoinedFlatTable { } public static String generateInsertDataStatement(IJoinedFlatTableDesc flatDesc) { - final KylinConfig kylinConfig = ((CubeSegment) flatDesc.getSegment()).getConfig(); + CubeSegment segment = ((CubeSegment) flatDesc.getSegment()); + KylinConfig kylinConfig; + if (null == segment) { + kylinConfig = KylinConfig.getInstanceFromEnv(); + } else { + kylinConfig = ((CubeSegment) flatDesc.getSegment()).getConfig(); + } if (kylinConfig.isAdvancedFlatTableUsed()) { try { Class advancedFlatTable = Class.forName(kylinConfig.getAdvancedFlatTableClass()); http://git-wip-us.apache.org/repos/asf/kylin/blob/0762c4dd/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableExtDesc.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableExtDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableExtDesc.java index 18fefc2..9692f5a 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableExtDesc.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableExtDesc.java @@ -49,9 +49,11 @@ public class TableExtDesc extends RootPersistentEntity { private List<String[]> sampleRows = new ArrayList<>(); @JsonProperty("last_modified_time") - private String lastModifiedTime; + private long lastModifiedTime; @JsonProperty("total_rows") - private String totalRows; + private long totalRows; + @JsonProperty("mapper_rows") + private List<Long> mapRecords = new ArrayList<>(); @JsonProperty("data_source_properties") private Map<String, String> dataSourceProps = new HashMap<>(); @@ -90,6 +92,14 @@ public class TableExtDesc extends RootPersistentEntity { return this.sampleRows; } + public void setMapRecords(List<Long> mapRecords) { + this.mapRecords = mapRecords; + } + + public List<Long> getMapRecords() { + return this.mapRecords; + } + public String getCardinality() { StringBuffer cardinality = new StringBuffer(); @@ -131,11 +141,11 @@ public class TableExtDesc extends RootPersistentEntity { this.columnStats = columnStats; } - public void setTotalRows(String totalRows) { + public void setTotalRows(long totalRows) { this.totalRows = totalRows; } - public String getTotalRows() { + public long getTotalRows() { return this.totalRows; } @@ -152,11 +162,11 @@ public class TableExtDesc extends RootPersistentEntity { this.tableName = this.tableName.toUpperCase(); } - public void setLastModifiedTime(String lastModifiedTime) { + public void setLastModifiedTime(long lastModifiedTime) { this.lastModifiedTime = lastModifiedTime; } - public String getLastModifiedTime() { + public long getLastModifiedTime() { return this.lastModifiedTime; } @@ -187,6 +197,9 @@ public class TableExtDesc extends RootPersistentEntity { @JsonBackReference private TableExtDesc tableExtDesc; + @JsonProperty("column_name") + private String columnName; + @JsonProperty("max_value") private String maxValue; @@ -205,6 +218,9 @@ public class TableExtDesc extends RootPersistentEntity { @JsonProperty("cardinality") private long cardinality; + @JsonProperty("data_skew_samples") + private Map<String, Long> dataSkewSamples = new HashMap<>(); + @Override public int compareTo(ColumnStats o) { return 0; @@ -213,6 +229,14 @@ public class TableExtDesc extends RootPersistentEntity { public ColumnStats() { } + public void setColumnName(String columnName) { + this.columnName = columnName; + } + + public String getColumnName() { + return this.columnName; + } + public void setMaxValue(String maxValue) { this.maxValue = maxValue; } @@ -253,6 +277,14 @@ public class TableExtDesc extends RootPersistentEntity { return this.cardinality; } + public void setDataSkewSamples(Map<String, Long> dataSkewSamples) { + this.dataSkewSamples = dataSkewSamples; + } + + public Map<String, Long> getDataSkewSamples() { + return this.dataSkewSamples; + } + public void setColumnSamples(String max, String min, String maxLenValue, String minLenValue) { this.maxValue = max; this.minValue = min;
