fix migration cli
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/b66c2580 Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/b66c2580 Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/b66c2580 Branch: refs/heads/devstreaming Commit: b66c25803a2f976cca067148278dbe7d7b0d79ef Parents: 54cab0a Author: honma <ho...@ebay.com> Authored: Wed Sep 16 14:37:42 2015 +0800 Committer: honma <ho...@ebay.com> Committed: Wed Sep 16 16:44:03 2015 +0800 ---------------------------------------------------------------------- .../org/apache/kylin/common/KylinConfig.java | 2 +- .../kylin/job/tools/CubeMigrationCLI.java | 19 +++- .../storage/hbase/steps/HBaseMROutput2.java | 1 + webapp/app/js/model/cubeDescModel.js | 104 ++++++++++--------- 4 files changed, 69 insertions(+), 57 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b66c2580/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java index db213f7..43b8c4d 100644 --- a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java +++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java @@ -502,7 +502,7 @@ public class KylinConfig implements Serializable { } public String getHbaseDefaultCompressionCodec() { - return getOptional(HTABLE_DEFAULT_COMPRESSION_CODEC); + return getOptional(HTABLE_DEFAULT_COMPRESSION_CODEC,""); } public boolean isHiveKeepFlatTable() { http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b66c2580/job/src/main/java/org/apache/kylin/job/tools/CubeMigrationCLI.java ---------------------------------------------------------------------- diff --git a/job/src/main/java/org/apache/kylin/job/tools/CubeMigrationCLI.java b/job/src/main/java/org/apache/kylin/job/tools/CubeMigrationCLI.java index 89d55f6..c68196a 100644 --- a/job/src/main/java/org/apache/kylin/job/tools/CubeMigrationCLI.java +++ b/job/src/main/java/org/apache/kylin/job/tools/CubeMigrationCLI.java @@ -24,6 +24,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; +import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -45,6 +46,7 @@ import org.apache.kylin.dict.lookup.SnapshotManager; import org.apache.kylin.dict.lookup.SnapshotTable; import org.apache.kylin.job.JobInstance; import org.apache.kylin.metadata.model.DataModelDesc; +import org.apache.kylin.metadata.model.IEngineAware; import org.apache.kylin.metadata.model.SegmentStatusEnum; import org.apache.kylin.metadata.model.TableDesc; import org.apache.kylin.metadata.project.ProjectInstance; @@ -87,7 +89,6 @@ public class CubeMigrationCLI { private static void usage() { System.out.println("Usage: CubeMigrationCLI srcKylinConfigUri dstKylinConfigUri cubeName projectName overwriteIfExists realExecute"); System.out.println(" srcKylinConfigUri: The KylinConfig of the cubeâs source \n" + "dstKylinConfigUri: The KylinConfig of the cubeâs new home \n" + "cubeName: the name of cube to be migrated. \n" + "projectName: The target project in the target environment.(Make sure it exist) \n" + "overwriteIfExists: overwrite cube if it already exists in the target environment. \n" + "realExecute: if false, just print the operations to take, if true, do the real migration. \n"); - } public static void moveCube(KylinConfig srcCfg, KylinConfig dstCfg, String cubeName, String projectName, String overwriteIfExists, String realExecute) throws IOException, InterruptedException { @@ -132,7 +133,6 @@ public class CubeMigrationCLI { } public static void moveCube(String srcCfgUri, String dstCfgUri, String cubeName, String projectName, String overwriteIfExists, String realExecute) throws IOException, InterruptedException { - moveCube(KylinConfig.createInstanceFromUri(srcCfgUri), KylinConfig.createInstanceFromUri(dstCfgUri), cubeName, projectName, overwriteIfExists, realExecute); } @@ -143,8 +143,8 @@ public class CubeMigrationCLI { logger.info("src metadata url is " + srcMetadataUrl); logger.info("dst metadata url is " + dstMetadataUrl); - int srcIndex = srcMetadataUrl.toLowerCase().indexOf("hbase:"); - int dstIndex = dstMetadataUrl.toLowerCase().indexOf("hbase:"); + int srcIndex = srcMetadataUrl.toLowerCase().indexOf("hbase"); + int dstIndex = dstMetadataUrl.toLowerCase().indexOf("hbase"); if (srcIndex < 0 || dstIndex < 0) throw new IllegalStateException("Both metadata urls should be hbase metadata url"); @@ -162,6 +162,11 @@ public class CubeMigrationCLI { for (CubeSegment segment : cube.getSegments()) { String jobUuid = segment.getLastBuildJobID(); + + if (StringUtils.isEmpty(jobUuid)) { + //segments build from streaming does not have hdfs working dir + continue; + } String src = JobInstance.getJobWorkingDir(jobUuid, srcConfig.getHdfsWorkingDirectory()); String tgt = JobInstance.getJobWorkingDir(jobUuid, dstConfig.getHdfsWorkingDirectory()); @@ -216,7 +221,11 @@ public class CubeMigrationCLI { for (CubeSegment segment : cube.getSegments()) { dictAndSnapshot.addAll(segment.getSnapshotPaths()); dictAndSnapshot.addAll(segment.getDictionaryPaths()); - metaResource.add(segment.getStatisticsResourcePath()); + + if (cube.getDescriptor().getEngineType() == IEngineAware.ID_MR_V2) { + //only V2 has this + metaResource.add(segment.getStatisticsResourcePath()); + } } } http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b66c2580/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMROutput2.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMROutput2.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMROutput2.java index 1e414be..79ef403 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMROutput2.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMROutput2.java @@ -170,6 +170,7 @@ public class HBaseMROutput2 implements IMROutput2 { scans.add(scan); } TableMapReduceUtil.initTableMapperJob(scans, (Class<? extends TableMapper>) mapper, outputKeyClass, outputValueClass, job); + TableMapReduceUtil.initCredentials(job); } @Override http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b66c2580/webapp/app/js/model/cubeDescModel.js ---------------------------------------------------------------------- diff --git a/webapp/app/js/model/cubeDescModel.js b/webapp/app/js/model/cubeDescModel.js index 86c8444..3376bfc 100644 --- a/webapp/app/js/model/cubeDescModel.js +++ b/webapp/app/js/model/cubeDescModel.js @@ -14,62 +14,64 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. -*/ + */ -KylinApp.service('CubeDescModel',function(){ +KylinApp.service('CubeDescModel', function () { - this.cubeMetaFrame = {}; + this.cubeMetaFrame = {}; - // - this.createNew = function () { - var cubeMeta = { - "name": "", - "model_name": "", - "description": "", - "dimensions": [], - "measures": [ - { "id": 1, - "name": "_COUNT_", - "function": { - "expression": "COUNT", - "returntype": "bigint", - "parameter": { - "type": "constant", - "value": "1" - } - } - } - ], - "rowkey": { - "rowkey_columns": [], - "aggregation_groups": [] - }, - "notify_list": [], - "hbase_mapping": { - "column_family": [] - }, - "retention_range":"0", - "auto_merge_time_ranges":[604800000,2419200000] - }; - - return cubeMeta; - }; + // + this.createNew = function () { + var cubeMeta = { + "name": "", + "model_name": "", + "description": "", + "dimensions": [], + "measures": [ + { + "id": 1, + "name": "_COUNT_", + "function": { + "expression": "COUNT", + "returntype": "bigint", + "parameter": { + "type": "constant", + "value": "1" + } + } + } + ], + "rowkey": { + "rowkey_columns": [], + "aggregation_groups": [] + }, + "notify_list": [], + "hbase_mapping": { + "column_family": [] + }, + "retention_range": "0", + "auto_merge_time_ranges": [604800000, 2419200000], + "engine_type": 2 + }; - this.createMeasure = function (){ - var measure = { - "id": "", - "name": "", - "function": { - "expression": "", - "returntype": "", - "parameter": { - "type": "", - "value": "" - } - } - }; + return cubeMeta; + }; - return measure; + this.createMeasure = function () { + var measure = { + "id": "", + "name": "", + "function": { + "expression": "", + "returntype": "", + "parameter": { + "type": "", + "value": "" } + } + }; + + return measure; + } })
