Repository: incubator-kylin Updated Branches: refs/heads/0.7-staging 2beba36ea -> 7a3941d81
KYLIN-926 clean all runtime files KYLIN-926 KYLIN-926 KYLIN-926 KYLIN-926 KYLIN-926 clean parent file KYLIN-926 cond KYLIN-926 KYLIN-926 fix /tmp/cardinality and minicluster tgz KYLIN-926 Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/7a3941d8 Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/7a3941d8 Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/7a3941d8 Branch: refs/heads/0.7-staging Commit: 7a3941d81c2f7df036c7d9f0a96045055730433f Parents: 2beba36 Author: honma <ho...@ebay.com> Authored: Mon Aug 17 16:32:10 2015 +0800 Committer: honma <ho...@ebay.com> Committed: Wed Aug 19 13:14:32 2015 +0800 ---------------------------------------------------------------------- .../org/apache/kylin/common/KylinConfig.java | 15 +- .../apache/kylin/common/util/HbaseImporter.java | 9 +- .../kylin/job/constant/BatchConstants.java | 33 ++-- .../kylin/job/hadoop/AbstractHadoopJob.java | 45 +++-- .../cardinality/HiveColumnCardinalityJob.java | 3 +- .../job/hadoop/cube/FactDistinctColumnsJob.java | 10 +- .../kylin/job/hadoop/cube/NDCuboidMapper.java | 4 +- .../job/hadoop/cube/NewBaseCuboidMapper.java | 2 +- .../kylin/job/hadoop/hbase/CreateHTableJob.java | 4 - .../apache/kylin/job/BuildIIWithEngineTest.java | 1 + .../java/org/apache/kylin/job/DeployUtil.java | 2 +- .../org/apache/kylin/job/ExportHBaseData.java | 41 +++-- .../org/apache/kylin/job/ImportHBaseData.java | 174 ------------------- .../job/hadoop/cube/MergeCuboidJobTest.java | 8 +- .../apache/kylin/query/test/IIQueryTest.java | 2 - .../apache/kylin/rest/service/CubeService.java | 3 - 16 files changed, 84 insertions(+), 272 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7a3941d8/common/src/main/java/org/apache/kylin/common/KylinConfig.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/kylin/common/KylinConfig.java b/common/src/main/java/org/apache/kylin/common/KylinConfig.java index 0c7b813..0e4cc07 100644 --- a/common/src/main/java/org/apache/kylin/common/KylinConfig.java +++ b/common/src/main/java/org/apache/kylin/common/KylinConfig.java @@ -52,15 +52,6 @@ public class KylinConfig { public static final String KYLIN_REST_SERVERS = "kylin.rest.servers"; public static final String KYLIN_REST_TIMEZONE = "kylin.rest.timezone"; - /** - * The dir containing scripts for kylin. For example: /usr/lib/kylin/bin - */ - public static final String KYLIN_SCRIPT_DIR = "kylin.script.dir"; - /** - * The script file name for generating table metadat from hive. For example: - * generateTable.sh - */ - public static final String KYLIN_SCRIPT_GEN_TABLE_META = "kylin.script.genTableMeta"; public static final String KYLIN_JOB_CONCURRENT_MAX_LIMIT = "kylin.job.concurrent.max.limit"; @@ -124,6 +115,7 @@ public class KylinConfig { public static final String MAIL_SENDER = "mail.sender"; public static final String KYLIN_HOME = "KYLIN_HOME"; + public static final String KYLIN_CONF = "KYLIN_CONF"; private static final Logger logger = LoggerFactory.getLogger(KylinConfig.class); @@ -233,8 +225,7 @@ public class KylinConfig { /** * Find config from environment. The Search process: 1. Check the - * $KYLIN_CONF/kylin.properties 2. Check the /etc/kylin/kylin.properties 3. - * Check the kylin.properties in classpath + * $KYLIN_CONF/kylin.properties 2. Check the $KYLIN_HOME/conf/kylin.properties * * @return */ @@ -284,7 +275,7 @@ public class KylinConfig { if (!root.endsWith("/")) { root += "/"; } - return root + getMetadataUrlPrefix(); + return root + getMetadataUrlPrefix() + "/"; } public String getKylinJobLogDir() { http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7a3941d8/common/src/test/java/org/apache/kylin/common/util/HbaseImporter.java ---------------------------------------------------------------------- diff --git a/common/src/test/java/org/apache/kylin/common/util/HbaseImporter.java b/common/src/test/java/org/apache/kylin/common/util/HbaseImporter.java index 0465be0..097272f 100644 --- a/common/src/test/java/org/apache/kylin/common/util/HbaseImporter.java +++ b/common/src/test/java/org/apache/kylin/common/util/HbaseImporter.java @@ -35,9 +35,6 @@ import org.apache.kylin.common.persistence.HBaseConnection; import com.google.common.base.Preconditions; -/** - * Created by Hongbin Ma(Binmahone) on 2/6/15. - */ public class HbaseImporter { private static final Log logger = LogFactory.getLog(HbaseImporter.class); @@ -56,12 +53,12 @@ public class HbaseImporter { return; } - File folder = new File("/tmp/hbase-export/"); + File folder = File.createTempFile("hbase-import", "tmp"); if (folder.exists()) { - FileUtils.deleteDirectory(folder); + FileUtils.forceDelete(folder); } folder.mkdirs(); - folder.deleteOnExit(); + FileUtils.forceDeleteOnExit(folder); //TarGZUtil.uncompressTarGZ(exportFile, folder); FileUtil.unTar(exportFile, folder); http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7a3941d8/job/src/main/java/org/apache/kylin/job/constant/BatchConstants.java ---------------------------------------------------------------------- diff --git a/job/src/main/java/org/apache/kylin/job/constant/BatchConstants.java b/job/src/main/java/org/apache/kylin/job/constant/BatchConstants.java index 0ac04fb..34c5760 100644 --- a/job/src/main/java/org/apache/kylin/job/constant/BatchConstants.java +++ b/job/src/main/java/org/apache/kylin/job/constant/BatchConstants.java @@ -24,29 +24,30 @@ package org.apache.kylin.job.constant; */ public interface BatchConstants { - public static final char INTERMEDIATE_TABLE_ROW_DELIMITER = 127; + char INTERMEDIATE_TABLE_ROW_DELIMITER = 127; - public static final String CFG_CUBE_NAME = "cube.name"; - public static final String CFG_CUBE_SEGMENT_NAME = "cube.segment.name"; + String CFG_CUBE_NAME = "cube.name"; + String CFG_CUBE_SEGMENT_NAME = "cube.segment.name"; - public static final String CFG_II_NAME = "ii.name"; - public static final String CFG_II_SEGMENT_NAME = "ii.segment.name"; + String CFG_II_NAME = "ii.name"; + String CFG_II_SEGMENT_NAME = "ii.segment.name"; - public static final String INPUT_DELIM = "input.delim"; + String OUTPUT_PATH = "output.path"; - public static final String OUTPUT_PATH = "output.path"; + String TABLE_NAME = "table.name"; + String TABLE_COLUMNS = "table.columns"; - public static final String TABLE_NAME = "table.name"; - public static final String TABLE_COLUMNS = "table.columns"; + String CFG_CUBE_INTERMEDIATE_TABLE_ROW_DELIMITER = "cube.intermediate.table.row.delimiter"; - public static final String CFG_CUBE_INTERMEDIATE_TABLE_ROW_DELIMITER = "cube.intermediate.table.row.delimiter"; + String MAPREDUCE_COUNTER_GROUP_NAME = "Cube Builder"; - public static final String MAPREDUCE_COUTNER_GROUP_NAME = "Cube Builder"; + String MAPPER_SAMPLE_NUMBER = "mapper.sample.number"; + String REGION_NUMBER = "region.number"; + String CUBE_CAPACITY = "cube.capacity"; - public static final String MAPPER_SAMPLE_NUMBER = "mapper.sample.number"; - public static final String REGION_NUMBER = "region.number"; - public static final String CUBE_CAPACITY = "cube.capacity"; + String CFG_KYLIN_LOCAL_TEMP_DIR = "/tmp/kylin/"; + String CFG_KYLIN_HDFS_TEMP_DIR = "/tmp/kylin/"; - public static final int COUNTER_MAX = 100000; - public static final int ERROR_RECORD_THRESHOLD = 100; + int COUNTER_MAX = 100000; + int ERROR_RECORD_THRESHOLD = 100; } http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7a3941d8/job/src/main/java/org/apache/kylin/job/hadoop/AbstractHadoopJob.java ---------------------------------------------------------------------- diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/AbstractHadoopJob.java b/job/src/main/java/org/apache/kylin/job/hadoop/AbstractHadoopJob.java index 61a8e48..4c60c52 100644 --- a/job/src/main/java/org/apache/kylin/job/hadoop/AbstractHadoopJob.java +++ b/job/src/main/java/org/apache/kylin/job/hadoop/AbstractHadoopJob.java @@ -36,6 +36,7 @@ import org.apache.commons.cli.Option; import org.apache.commons.cli.OptionBuilder; import org.apache.commons.cli.Options; import org.apache.commons.cli.ParseException; +import org.apache.commons.io.FileUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configured; import org.apache.hadoop.fs.FileStatus; @@ -48,7 +49,6 @@ import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; import org.apache.hadoop.util.ReflectionUtils; import org.apache.hadoop.util.Tool; -import org.apache.hadoop.util.ToolRunner; import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.persistence.ResourceStore; import org.apache.kylin.common.util.CliCommandExecutor; @@ -82,11 +82,8 @@ public abstract class AbstractHadoopJob extends Configured implements Tool { protected static final Option OPTION_NCUBOID_LEVEL = OptionBuilder.withArgName("level").hasArg().isRequired(true).withDescription("N-Cuboid build level, e.g. 1, 2, 3...").create("level"); protected static final Option OPTION_PARTITION_FILE_PATH = OptionBuilder.withArgName("path").hasArg().isRequired(true).withDescription("Partition file path.").create("input"); protected static final Option OPTION_HTABLE_NAME = OptionBuilder.withArgName("htable name").hasArg().isRequired(true).withDescription("HTable name").create("htablename"); - protected static final Option OPTION_KEY_COLUMN_PERCENTAGE = OptionBuilder.withArgName("rowkey column percentage").hasArg().isRequired(true).withDescription("Percentage of row key columns").create("columnpercentage"); - protected static final Option OPTION_KEY_SPLIT_NUMBER = OptionBuilder.withArgName("key split number").hasArg().isRequired(true).withDescription("Number of key split range").create("splitnumber"); protected String name; - protected String description; protected boolean isAsync = false; protected OptionsHelper optionsHelper = new OptionsHelper(); @@ -129,16 +126,6 @@ public abstract class AbstractHadoopJob extends Configured implements Tool { return retVal; } - protected static void runJob(Tool job, String[] args) { - try { - int exitCode = ToolRunner.run(job, args); - System.exit(exitCode); - } catch (Exception e) { - e.printStackTrace(System.err); - System.exit(5); - } - } - private static final String MAP_REDUCE_CLASSPATH = "mapreduce.application.classpath"; protected void setJobClasspath(Job job) { @@ -222,7 +209,7 @@ public abstract class AbstractHadoopJob extends Configured implements Tool { protected void attachKylinPropsAndMetadata(CubeInstance cube, Configuration conf) throws IOException { File tmp = File.createTempFile("kylin_job_meta", ""); - tmp.delete(); // we need a directory, so delete the file first + FileUtils.forceDelete(tmp); File metaDir = new File(tmp, "meta"); metaDir.mkdirs(); @@ -249,29 +236,41 @@ public abstract class AbstractHadoopJob extends Configured implements Tool { dumpResources(kylinConfig, metaDir, dumpList); // hadoop distributed cache - String hdfsMetaDir = "file:///" + OptionsHelper.convertToFileURL(metaDir.getAbsolutePath()); + String hdfsMetaDir = "file://" + OptionsHelper.convertToFileURL(metaDir.getAbsolutePath()); logger.info("HDFS meta dir is: " + hdfsMetaDir); conf.set("tmpfiles", hdfsMetaDir); - + } protected void cleanupTempConfFile(Configuration conf) { String tempMetaFileString = conf.get("tmpfiles"); + logger.info("tempMetaFileString is : " + tempMetaFileString); if (tempMetaFileString != null) { - File tempMetaFile = new File(tempMetaFileString); - if (tempMetaFile.exists()) { - tempMetaFile.getParentFile().delete(); + if (tempMetaFileString.startsWith("file://")) { + tempMetaFileString = tempMetaFileString.substring("file://".length()); + File tempMetaFile = new File(tempMetaFileString); + if (tempMetaFile.exists()) { + try { + FileUtils.forceDelete(tempMetaFile.getParentFile()); + + } catch (IOException e) { + logger.warn("error when deleting " + tempMetaFile, e); + } + } else { + logger.info("" + tempMetaFileString + " does not exist"); + } + } else { + logger.info("tempMetaFileString is not starting with file:// :" + tempMetaFileString); } } } protected void attachKylinPropsAndMetadata(IIInstance ii, Configuration conf) throws IOException { File tmp = File.createTempFile("kylin_job_meta", ""); - tmp.delete(); // we need a directory, so delete the file first + FileUtils.forceDelete(tmp); File metaDir = new File(tmp, "meta"); metaDir.mkdirs(); - metaDir.getParentFile().deleteOnExit(); // write kylin.properties KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv(); @@ -296,7 +295,7 @@ public abstract class AbstractHadoopJob extends Configured implements Tool { dumpResources(kylinConfig, metaDir, dumpList); // hadoop distributed cache - String hdfsMetaDir = "file:///" + OptionsHelper.convertToFileURL(metaDir.getAbsolutePath()); + String hdfsMetaDir = "file://" + OptionsHelper.convertToFileURL(metaDir.getAbsolutePath()); logger.info("HDFS meta dir is: " + hdfsMetaDir); conf.set("tmpfiles", hdfsMetaDir); } http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7a3941d8/job/src/main/java/org/apache/kylin/job/hadoop/cardinality/HiveColumnCardinalityJob.java ---------------------------------------------------------------------- diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/cardinality/HiveColumnCardinalityJob.java b/job/src/main/java/org/apache/kylin/job/hadoop/cardinality/HiveColumnCardinalityJob.java index e47e05a..ed0abe7 100644 --- a/job/src/main/java/org/apache/kylin/job/hadoop/cardinality/HiveColumnCardinalityJob.java +++ b/job/src/main/java/org/apache/kylin/job/hadoop/cardinality/HiveColumnCardinalityJob.java @@ -31,6 +31,7 @@ import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat; import org.apache.hive.hcatalog.mapreduce.HCatInputFormat; import org.apache.kylin.common.util.HadoopUtil; +import org.apache.kylin.job.constant.BatchConstants; import org.apache.kylin.job.hadoop.AbstractHadoopJob; /** @@ -44,7 +45,7 @@ public class HiveColumnCardinalityJob extends AbstractHadoopJob { @SuppressWarnings("static-access") protected static final Option OPTION_TABLE = OptionBuilder.withArgName("table name").hasArg().isRequired(true).withDescription("The hive table name").create("table"); - public static final String OUTPUT_PATH = "/tmp/cardinality"; + public static final String OUTPUT_PATH = BatchConstants.CFG_KYLIN_HDFS_TEMP_DIR + "cardinality"; public HiveColumnCardinalityJob() { } http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7a3941d8/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsJob.java ---------------------------------------------------------------------- diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsJob.java b/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsJob.java index 6e2b467..e13e011 100644 --- a/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsJob.java +++ b/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsJob.java @@ -18,29 +18,28 @@ package org.apache.kylin.job.hadoop.cube; +import java.io.IOException; + import org.apache.commons.cli.Options; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.NullWritable; import org.apache.hadoop.io.ShortWritable; import org.apache.hadoop.io.Text; +import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat; -import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.util.ToolRunner; import org.apache.hive.hcatalog.mapreduce.HCatInputFormat; import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.util.HadoopUtil; import org.apache.kylin.cube.CubeInstance; import org.apache.kylin.cube.CubeManager; -import org.apache.kylin.cube.model.CubeDesc; import org.apache.kylin.job.constant.BatchConstants; import org.apache.kylin.job.hadoop.AbstractHadoopJob; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; - /** * @author yangli9 */ @@ -88,8 +87,9 @@ public class FactDistinctColumnsJob extends AbstractHadoopJob { printUsage(options); throw e; } finally { - if (job != null) + if (job != null) { cleanupTempConfFile(job.getConfiguration()); + } } } http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7a3941d8/job/src/main/java/org/apache/kylin/job/hadoop/cube/NDCuboidMapper.java ---------------------------------------------------------------------- diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/cube/NDCuboidMapper.java b/job/src/main/java/org/apache/kylin/job/hadoop/cube/NDCuboidMapper.java index 7a69c91..f846232 100644 --- a/job/src/main/java/org/apache/kylin/job/hadoop/cube/NDCuboidMapper.java +++ b/job/src/main/java/org/apache/kylin/job/hadoop/cube/NDCuboidMapper.java @@ -115,7 +115,7 @@ public class NDCuboidMapper extends KylinMapper<Text, Text, Text, Text> { // if still empty or null if (myChildren == null || myChildren.size() == 0) { - context.getCounter(BatchConstants.MAPREDUCE_COUTNER_GROUP_NAME, "Skipped records").increment(1L); + context.getCounter(BatchConstants.MAPREDUCE_COUNTER_GROUP_NAME, "Skipped records").increment(1L); skipCounter++; if (skipCounter % BatchConstants.COUNTER_MAX == 0) { logger.info("Skipped " + skipCounter + " records!"); @@ -123,7 +123,7 @@ public class NDCuboidMapper extends KylinMapper<Text, Text, Text, Text> { return; } - context.getCounter(BatchConstants.MAPREDUCE_COUTNER_GROUP_NAME, "Processed records").increment(1L); + context.getCounter(BatchConstants.MAPREDUCE_COUNTER_GROUP_NAME, "Processed records").increment(1L); handleCounter++; if (handleCounter % BatchConstants.COUNTER_MAX == 0) { http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7a3941d8/job/src/main/java/org/apache/kylin/job/hadoop/cube/NewBaseCuboidMapper.java ---------------------------------------------------------------------- diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/cube/NewBaseCuboidMapper.java b/job/src/main/java/org/apache/kylin/job/hadoop/cube/NewBaseCuboidMapper.java index b6e4233..e5c891b 100644 --- a/job/src/main/java/org/apache/kylin/job/hadoop/cube/NewBaseCuboidMapper.java +++ b/job/src/main/java/org/apache/kylin/job/hadoop/cube/NewBaseCuboidMapper.java @@ -339,7 +339,7 @@ public class NewBaseCuboidMapper<KEYIN> extends KylinMapper<KEYIN, Text, Text, T } catch (Throwable t) { logger.error("", t); - context.getCounter(BatchConstants.MAPREDUCE_COUTNER_GROUP_NAME, "Error records").increment(1L); + context.getCounter(BatchConstants.MAPREDUCE_COUNTER_GROUP_NAME, "Error records").increment(1L); return; } } http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7a3941d8/job/src/main/java/org/apache/kylin/job/hadoop/hbase/CreateHTableJob.java ---------------------------------------------------------------------- diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/hbase/CreateHTableJob.java b/job/src/main/java/org/apache/kylin/job/hadoop/hbase/CreateHTableJob.java index e9e30c2..1350077 100644 --- a/job/src/main/java/org/apache/kylin/job/hadoop/hbase/CreateHTableJob.java +++ b/job/src/main/java/org/apache/kylin/job/hadoop/hbase/CreateHTableJob.java @@ -53,10 +53,6 @@ import org.apache.kylin.metadata.realization.IRealizationConstants; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -/** - * @author George Song (ysong1) - */ - public class CreateHTableJob extends AbstractHadoopJob { protected static final Logger logger = LoggerFactory.getLogger(CreateHTableJob.class); http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7a3941d8/job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java ---------------------------------------------------------------------- diff --git a/job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java b/job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java index d39a27e..3916c40 100644 --- a/job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java +++ b/job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java @@ -237,6 +237,7 @@ public class BuildIIWithEngineTest { private void exportHBaseData() throws IOException { ExportHBaseData export = new ExportHBaseData(); export.exportTables(); + export.tearDown(); } public static void main(String[] args) throws Exception { http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7a3941d8/job/src/test/java/org/apache/kylin/job/DeployUtil.java ---------------------------------------------------------------------- diff --git a/job/src/test/java/org/apache/kylin/job/DeployUtil.java b/job/src/test/java/org/apache/kylin/job/DeployUtil.java index d1c9745..41399c7 100644 --- a/job/src/test/java/org/apache/kylin/job/DeployUtil.java +++ b/job/src/test/java/org/apache/kylin/job/DeployUtil.java @@ -24,6 +24,7 @@ import java.io.FileReader; import java.io.IOException; import java.io.InputStream; +import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; import org.apache.commons.lang.StringUtils; import org.apache.kylin.common.KylinConfig; @@ -43,7 +44,6 @@ import org.apache.kylin.metadata.model.TableDesc; import org.apache.maven.model.Model; import org.apache.maven.model.io.xpp3.MavenXpp3Reader; import org.apache.tools.ant.filters.StringInputStream; -import org.codehaus.plexus.util.FileUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7a3941d8/job/src/test/java/org/apache/kylin/job/ExportHBaseData.java ---------------------------------------------------------------------- diff --git a/job/src/test/java/org/apache/kylin/job/ExportHBaseData.java b/job/src/test/java/org/apache/kylin/job/ExportHBaseData.java index 9e302b9..557dd39 100644 --- a/job/src/test/java/org/apache/kylin/job/ExportHBaseData.java +++ b/job/src/test/java/org/apache/kylin/job/ExportHBaseData.java @@ -21,6 +21,7 @@ package org.apache.kylin.job; import java.io.File; import java.io.IOException; +import org.apache.commons.io.FileUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.client.HBaseAdmin; @@ -31,7 +32,7 @@ import org.apache.kylin.common.util.AbstractKylinTestCase; import org.apache.kylin.common.util.CliCommandExecutor; import org.apache.kylin.common.util.HBaseMiniclusterHelper; import org.apache.kylin.common.util.SSHClient; -import org.codehaus.plexus.util.FileUtils; +import org.apache.kylin.job.constant.BatchConstants; public class ExportHBaseData { @@ -39,10 +40,13 @@ public class ExportHBaseData { HTableDescriptor[] allTables; Configuration config; HBaseAdmin hbase; - CliCommandExecutor cli = null; - String exportFolder; - String backupArchive = null; + CliCommandExecutor cli; + String exportHdfsFolder; + String exportLocalFolderParent; + String exportLocalFolder; + String backupArchive; String tableNameBase; + long currentTIME; public ExportHBaseData() { try { @@ -53,9 +57,6 @@ public class ExportHBaseData { } private void setup() throws IOException { - long currentTIME = System.currentTimeMillis(); - exportFolder = "/tmp/hbase-export/" + currentTIME + "/"; - backupArchive = "/tmp/kylin_" + currentTIME + ".tar.gz"; KylinConfig.destoryInstance(); System.setProperty(KylinConfig.KYLIN_CONF, AbstractKylinTestCase.SANDBOX_TEST_DATA); @@ -63,6 +64,12 @@ public class ExportHBaseData { kylinConfig = KylinConfig.getInstanceFromEnv(); cli = kylinConfig.getCliCommandExecutor(); + currentTIME = System.currentTimeMillis(); + exportHdfsFolder = kylinConfig.getHdfsWorkingDirectory() + "hbase-export/" + currentTIME + "/"; + exportLocalFolderParent = BatchConstants.CFG_KYLIN_LOCAL_TEMP_DIR + "hbase-export/"; + exportLocalFolder = exportLocalFolderParent + currentTIME + "/"; + backupArchive = exportLocalFolderParent + "hbase-export-at-" + currentTIME + ".tar.gz"; + String metadataUrl = kylinConfig.getMetadataUrl(); // split TABLE@HBASE_URL int cut = metadataUrl.indexOf('@'); @@ -84,16 +91,16 @@ public class ExportHBaseData { // cleanup hdfs try { - if (cli != null && exportFolder != null) { - cli.execute("hadoop fs -rm -r " + exportFolder); + if (cli != null && exportHdfsFolder != null) { + cli.execute("hadoop fs -rm -r " + exportHdfsFolder); } } catch (IOException e) { e.printStackTrace(); } // cleanup sandbox disk try { - if (cli != null && exportFolder != null) { - cli.execute("rm -r " + exportFolder); + if (cli != null && exportLocalFolder != null) { + cli.execute("rm -r " + exportLocalFolder); } } catch (IOException e) { e.printStackTrace(); @@ -107,24 +114,22 @@ public class ExportHBaseData { } catch (IOException e) { e.printStackTrace(); } - - KylinConfig.destoryInstance(); - } public void exportTables() throws IOException { - cli.execute("mkdir -p " + exportFolder); + cli.execute("mkdir -p " + exportLocalFolderParent); for (HTableDescriptor table : allTables) { String tName = table.getNameAsString(); if (!tName.equals(tableNameBase) && !tName.startsWith(HBaseMiniclusterHelper.SHARED_STORAGE_PREFIX)) continue; - cli.execute("hbase org.apache.hadoop.hbase.mapreduce.Export " + tName + " " + exportFolder + tName); + cli.execute("hbase org.apache.hadoop.hbase.mapreduce.Export " + tName + " " + exportHdfsFolder + tName); } - cli.execute("hadoop fs -copyToLocal " + exportFolder + " " + exportFolder); - cli.execute("tar -zcvf " + backupArchive + " --directory=" + exportFolder + " ."); + cli.execute("hadoop fs -copyToLocal " + exportHdfsFolder + " " + exportLocalFolderParent); + cli.execute("tar -zcvf " + backupArchive + " --directory=" + exportLocalFolderParent + " " + currentTIME); + downloadToLocal(); } http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7a3941d8/job/src/test/java/org/apache/kylin/job/ImportHBaseData.java ---------------------------------------------------------------------- diff --git a/job/src/test/java/org/apache/kylin/job/ImportHBaseData.java b/job/src/test/java/org/apache/kylin/job/ImportHBaseData.java deleted file mode 100644 index 60983b6..0000000 --- a/job/src/test/java/org/apache/kylin/job/ImportHBaseData.java +++ /dev/null @@ -1,174 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.job; - -import java.io.File; -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HTableDescriptor; -import org.apache.hadoop.hbase.client.HBaseAdmin; -import org.apache.hadoop.hbase.client.HConnection; -import org.apache.kylin.common.KylinConfig; -import org.apache.kylin.common.persistence.HBaseConnection; -import org.apache.kylin.common.persistence.HBaseResourceStore; -import org.apache.kylin.common.util.AbstractKylinTestCase; -import org.apache.kylin.common.util.CliCommandExecutor; -import org.apache.kylin.common.util.HBaseMiniclusterHelper; -import org.apache.kylin.common.util.SSHClient; -import org.apache.kylin.common.util.TarGZUtil; - -public class ImportHBaseData { - - KylinConfig kylinConfig; - HTableDescriptor[] allTables; - Configuration config; - HBaseAdmin hbase; - CliCommandExecutor cli = null; - String importFolder = "/tmp/hbase-export/"; - String backupArchive = null; - String tableNameBase; - - public void setup() throws IOException { - - KylinConfig.destoryInstance(); - System.setProperty(KylinConfig.KYLIN_CONF, AbstractKylinTestCase.SANDBOX_TEST_DATA); - - kylinConfig = KylinConfig.getInstanceFromEnv(); - cli = kylinConfig.getCliCommandExecutor(); - - String metadataUrl = kylinConfig.getMetadataUrl(); - // split TABLE@HBASE_URL - int cut = metadataUrl.indexOf('@'); - tableNameBase = metadataUrl.substring(0, cut); - String hbaseUrl = cut < 0 ? metadataUrl : metadataUrl.substring(cut + 1); - - HConnection conn = HBaseConnection.get(hbaseUrl); - try { - hbase = new HBaseAdmin(conn); - config = hbase.getConfiguration(); - //allTables = hbase.listTables(); - } catch (IOException e) { - e.printStackTrace(); - throw e; - } - - uploadTarballToRemote(); - } - - public void tearDown() { - // cleanup sandbox disk - try { - if (cli != null && importFolder != null) { - cli.execute("rm -r " + importFolder); - } - } catch (IOException e) { - e.printStackTrace(); - } - - KylinConfig.destoryInstance(); - - } - - public void importTables() throws IOException { - // create the metadata htables - new HBaseResourceStore(KylinConfig.getInstanceFromEnv()); - - List<String> tablelocations = getTablesBackupLocations(importFolder); - for (String tableLocation : tablelocations) { - String table = tableLocation.substring(tableLocation.lastIndexOf("/") + 1); - - if (!(table.equalsIgnoreCase(tableNameBase) || table.startsWith(HBaseMiniclusterHelper.SHARED_STORAGE_PREFIX))) { - continue; - } - - if (table.startsWith(HBaseMiniclusterHelper.SHARED_STORAGE_PREFIX)) { - // create the cube table; otherwise the import will fail. - HBaseConnection.createHTableIfNeeded(KylinConfig.getInstanceFromEnv().getStorageUrl(), table, "F1", "F2"); - } - cli.execute("hbase org.apache.hadoop.hbase.mapreduce.Import " + table + " file://" + tableLocation); - } - - } - - public void uploadTarballToRemote() throws IOException { - - cli.execute("mkdir -p /tmp/hbase-export/"); - @SuppressWarnings("unused") - SSHClient ssh = new SSHClient(kylinConfig.getRemoteHadoopCliHostname(), kylinConfig.getRemoteHadoopCliUsername(), kylinConfig.getRemoteHadoopCliPassword()); - try { - // ssh.scpFileToRemote("../examples/test_case_data/minicluster/hbase-export.tar.gz", importFolder); - } catch (Exception e) { - e.printStackTrace(); - } - - cli.execute("tar -xzf /tmp/hbase-export/hbase-export.tar.gz --directory=" + importFolder); - } - - private List<String> getTablesBackupLocations(String exportBase) throws IOException { - File exportFile = new File("../examples/test_case_data/minicluster/hbase-export.tar.gz"); - - if (!exportFile.exists()) { - return null; - } - - File folder = new File("/tmp/hbase-export/"); - - if (folder.exists()) { - folder.delete(); - } - - folder.mkdirs(); - folder.deleteOnExit(); - - TarGZUtil.uncompressTarGZ(exportFile, folder); - - String[] child = folder.list(); - - assert child.length == 1; - - String backupTime = child[0]; - - File backupFolder = new File(folder, backupTime); - - String[] tableNames = backupFolder.list(); - - List<String> locations = new ArrayList<String>(15); - - for (String t : tableNames) { - locations.add(exportBase + backupTime + "/" + t); - } - - return locations; - } - - public static void main(String[] args) { - ImportHBaseData export = new ImportHBaseData(); - try { - export.setup(); - export.importTables(); - } catch (IOException e) { - e.printStackTrace(); - } finally { - //export.tearDown(); - } - } -} http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7a3941d8/job/src/test/java/org/apache/kylin/job/hadoop/cube/MergeCuboidJobTest.java ---------------------------------------------------------------------- diff --git a/job/src/test/java/org/apache/kylin/job/hadoop/cube/MergeCuboidJobTest.java b/job/src/test/java/org/apache/kylin/job/hadoop/cube/MergeCuboidJobTest.java index f9d4f45..72f515a 100644 --- a/job/src/test/java/org/apache/kylin/job/hadoop/cube/MergeCuboidJobTest.java +++ b/job/src/test/java/org/apache/kylin/job/hadoop/cube/MergeCuboidJobTest.java @@ -60,16 +60,16 @@ public class MergeCuboidJobTest extends LocalFileMetadataTestCase { String jobname = "merge_cuboid"; File baseFolder = File.createTempFile("kylin-f24668f6-dcff-4cb6-a89b-77f1119df8fa-", "base"); - baseFolder.delete(); + FileUtils.forceDelete(baseFolder); baseFolder.mkdir(); FileUtils.copyDirectory(new File("src/test/resources/data/base_cuboid"), baseFolder); - baseFolder.deleteOnExit(); + FileUtils.forceDeleteOnExit(baseFolder); File eightFolder = File.createTempFile("kylin-f24668f6-dcff-4cb6-a89b-77f1119df8fa-", "8d"); - eightFolder.delete(); + FileUtils.forceDelete(eightFolder); eightFolder.mkdir(); FileUtils.copyDirectory(new File("src/test/resources/data/base_cuboid"), eightFolder); - eightFolder.deleteOnExit(); + FileUtils.forceDeleteOnExit(eightFolder); FileUtil.fullyDelete(new File(output)); http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7a3941d8/query/src/test/java/org/apache/kylin/query/test/IIQueryTest.java ---------------------------------------------------------------------- diff --git a/query/src/test/java/org/apache/kylin/query/test/IIQueryTest.java b/query/src/test/java/org/apache/kylin/query/test/IIQueryTest.java index 198fb2f..80ff4bb 100644 --- a/query/src/test/java/org/apache/kylin/query/test/IIQueryTest.java +++ b/query/src/test/java/org/apache/kylin/query/test/IIQueryTest.java @@ -18,7 +18,6 @@ package org.apache.kylin.query.test; -import java.io.File; import java.util.Map; import org.apache.kylin.metadata.realization.RealizationType; @@ -59,5 +58,4 @@ public class IIQueryTest extends KylinQueryTest { execAndCompQuery("src/test/resources/query/sql_ii", null, true); } - } http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7a3941d8/server/src/main/java/org/apache/kylin/rest/service/CubeService.java ---------------------------------------------------------------------- diff --git a/server/src/main/java/org/apache/kylin/rest/service/CubeService.java b/server/src/main/java/org/apache/kylin/rest/service/CubeService.java index bd81c45..c3e6694 100644 --- a/server/src/main/java/org/apache/kylin/rest/service/CubeService.java +++ b/server/src/main/java/org/apache/kylin/rest/service/CubeService.java @@ -313,7 +313,6 @@ public class CubeService extends BasicService { * @param cube * @return * @throws IOException - * @throws CubeIntegrityException * @throws JobException */ @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN + " or hasPermission(#cube, 'ADMINISTRATION') or hasPermission(#cube, 'OPERATION') or hasPermission(#cube, 'MANAGEMENT')") @@ -339,7 +338,6 @@ public class CubeService extends BasicService { * Update a cube status from ready to disabled. * * @return - * @throws CubeIntegrityException * @throws IOException * @throws JobException */ @@ -528,7 +526,6 @@ public class CubeService extends BasicService { * * @throws IOException * @throws JobException - * @throws CubeIntegrityException */ private void releaseAllSegments(CubeInstance cube) throws IOException, JobException { final List<CubingJob> cubingJobs = listAllCubingJobs(cube.getName(), null);
