http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/main/java/org/apache/kylin/common/KylinConfig.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/kylin/common/KylinConfig.java b/common/src/main/java/org/apache/kylin/common/KylinConfig.java deleted file mode 100644 index 59cb86b..0000000 --- a/common/src/main/java/org/apache/kylin/common/KylinConfig.java +++ /dev/null @@ -1,775 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.common; - -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.io.InputStream; -import java.util.SortedSet; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import org.apache.commons.configuration.ConfigurationException; -import org.apache.commons.configuration.PropertiesConfiguration; -import org.apache.commons.io.IOUtils; -import org.apache.commons.lang.StringUtils; -import org.apache.kylin.common.restclient.RestClient; -import org.apache.kylin.common.util.CliCommandExecutor; -import org.apache.kylin.common.util.Log4jConfigurer; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.google.common.collect.Sets; - -/** - * @author yangli9 - */ -public class KylinConfig { - - /* - * NO NEED TO DEFINE PUBLIC CONSTANTS FOR KEY NAMES! - * - * Kylin configuration is exposed as public methods. A client never need to access key names directly. - * Feel free to hard code key names, as long as they are encapsulated in this class. It reads better! - */ - - public static final String KYLIN_STORAGE_URL = "kylin.storage.url"; - - public static final String KYLIN_METADATA_URL = "kylin.metadata.url"; - - public static final String KYLIN_REST_SERVERS = "kylin.rest.servers"; - - public static final String KYLIN_REST_TIMEZONE = "kylin.rest.timezone"; - - public static final String KYLIN_JOB_CONCURRENT_MAX_LIMIT = "kylin.job.concurrent.max.limit"; - - public static final String KYLIN_JOB_YARN_APP_REST_CHECK_URL = "kylin.job.yarn.app.rest.check.status.url"; - - public static final String KYLIN_JOB_YARN_APP_REST_CHECK_INTERVAL_SECONDS = "kylin.job.yarn.app.rest.check.interval.seconds"; - - public static final String KYLIN_TMP_HDFS_DIR = "kylin.tmp.hdfs.dir"; - - public static final String HIVE_DATABASE_FOR_INTERMEDIATE_TABLE = "kylin.job.hive.database.for.intermediatetable"; - - public static final String HIVE_TABLE_LOCATION_PREFIX = "hive.table.location."; - - public static final String KYLIN_JOB_REMOTE_CLI_PASSWORD = "kylin.job.remote.cli.password"; - - public static final String KYLIN_JOB_REMOTE_CLI_USERNAME = "kylin.job.remote.cli.username"; - - public static final String KYLIN_JOB_REMOTE_CLI_HOSTNAME = "kylin.job.remote.cli.hostname"; - - public static final String KYLIN_JOB_REMOTE_CLI_WORKING_DIR = "kylin.job.remote.cli.working.dir"; - - public static final String KYLIN_JOB_CMD_EXTRA_ARGS = "kylin.job.cmd.extra.args"; - - public static final String KYLIN_GET_JOB_STATUS_WITH_KERBEROS = "kylin.job.status.with.kerberos"; - /** - * Toggle to indicate whether to use hive for table flattening. Default - * true. - */ - public static final String KYLIN_JOB_RUN_AS_REMOTE_CMD = "kylin.job.run.as.remote.cmd"; - - public static final String KYLIN_JOB_MAPREDUCE_DEFAULT_REDUCE_COUNT_RATIO = "kylin.job.mapreduce.default.reduce.count.ratio"; - - public static final String KYLIN_JOB_MAPREDUCE_DEFAULT_REDUCE_INPUT_MB = "kylin.job.mapreduce.default.reduce.input.mb"; - - public static final String KYLIN_JOB_MAPREDUCE_MAX_REDUCER_NUMBER = "kylin.job.mapreduce.max.reducer.number"; - - public static final String KYLIN_JOB_JAR = "kylin.job.jar"; - - public static final String COPROCESSOR_LOCAL_JAR = "kylin.coprocessor.local.jar"; - - public static final String KYLIN_JOB_LOG_DIR = "kylin.job.log.dir"; - - public static final String KYLIN_HDFS_WORKING_DIR = "kylin.hdfs.working.dir"; - - public static final String KYLIN_HBASE_CLUSTER_FS = "kylin.hbase.cluster.fs"; - - /** - * Kylin properties file - */ - public static final String KYLIN_CONF_PROPERTIES_FILE = "kylin.properties"; - - public static final String MAIL_ENABLED = "mail.enabled"; - - public static final String MAIL_HOST = "mail.host"; - - public static final String MAIL_USERNAME = "mail.username"; - - public static final String MAIL_PASSWORD = "mail.password"; - - public static final String MAIL_SENDER = "mail.sender"; - - public static final String KYLIN_HOME = "KYLIN_HOME"; - - public static final String KYLIN_CONF = "KYLIN_CONF"; - - private static final Logger logger = LoggerFactory.getLogger(KylinConfig.class); - - public static final String VERSION = "${project.version}"; - - public static final String HTABLE_DEFAULT_COMPRESSION_CODEC = "kylin.hbase.default.compression.codec"; - - public static final String HBASE_REGION_CUT_SMALL = "kylin.hbase.region.cut.small"; - public static final String HBASE_REGION_CUT_MEDIUM = "kylin.hbase.region.cut.medium"; - public static final String HBASE_REGION_CUT_LARGE = "kylin.hbase.region.cut.large"; - - public static final String HBASE_REGION_COUNT_MIN = "kylin.hbase.region.count.min"; - public static final String HBASE_REGION_COUNT_MAX = "kylin.hbase.region.count.max"; - - // static cached instances - private static KylinConfig ENV_INSTANCE = null; - - public static KylinConfig getInstanceFromEnv() { - if (ENV_INSTANCE == null) { - try { - KylinConfig config = loadKylinConfig(); - ENV_INSTANCE = config; - } catch (IllegalArgumentException e) { - throw new IllegalStateException("Failed to find KylinConfig ", e); - } - } - return ENV_INSTANCE; - } - - public static void destoryInstance() { - ENV_INSTANCE = null; - } - - public static enum UriType { - PROPERTIES_FILE, REST_ADDR, LOCAL_FOLDER - } - - private static UriType decideUriType(String metaUri) { - - try { - File file = new File(metaUri); - if (file.exists() || metaUri.contains("/")) { - if (file.exists() == false) { - file.mkdirs(); - } - if (file.isDirectory()) { - return UriType.LOCAL_FOLDER; - } else if (file.isFile()) { - if (file.getName().equalsIgnoreCase(KYLIN_CONF_PROPERTIES_FILE)) { - return UriType.PROPERTIES_FILE; - } else { - throw new IllegalStateException("Metadata uri : " + metaUri + " is a local file but not kylin.properties"); - } - } else { - throw new IllegalStateException("Metadata uri : " + metaUri + " looks like a file but it's neither a file nor a directory"); - } - } else { - if (RestClient.matchFullRestPattern(metaUri)) - return UriType.REST_ADDR; - else - throw new IllegalStateException("Metadata uri : " + metaUri + " is not a valid REST URI address"); - } - } catch (Exception e) { - logger.info(e.getLocalizedMessage()); - throw new IllegalStateException("Metadata uri : " + metaUri + " is not recognized"); - } - } - - public static KylinConfig createInstanceFromUri(String uri) { - /** - * --hbase: 1. PROPERTIES_FILE: path to kylin.properties 2. REST_ADDR: - * rest service resource, format: user:password@host:port --local: 1. - * LOCAL_FOLDER: path to resource folder - */ - UriType uriType = decideUriType(uri); - logger.info("The URI " + uri + " is recognized as " + uriType); - - if (uriType == UriType.LOCAL_FOLDER) { - KylinConfig config = new KylinConfig(); - config.setMetadataUrl(uri); - return config; - } else if (uriType == UriType.PROPERTIES_FILE) { - KylinConfig config; - try { - config = new KylinConfig(); - InputStream is = new FileInputStream(uri); - config.reloadKylinConfig(is); - is.close(); - } catch (IOException e) { - throw new RuntimeException(e); - } - return config; - } else {// rest_addr - try { - KylinConfig config = new KylinConfig(); - RestClient client = new RestClient(uri); - String propertyText = client.getKylinProperties(); - InputStream is = IOUtils.toInputStream(propertyText); - config.reloadKylinConfig(is); - is.close(); - return config; - } catch (IOException e) { - throw new RuntimeException(e); - } - } - } - - public static KylinConfig getKylinConfigFromInputStream(InputStream is) { - KylinConfig config = new KylinConfig(); - config.reloadKylinConfig(is); - return config; - } - - // ============================================================================ - - /** - * Find config from environment. The Search process: 1. Check the - * $KYLIN_CONF/kylin.properties 2. Check the $KYLIN_HOME/conf/kylin.properties - * - * @return - */ - private static KylinConfig loadKylinConfig() { - Log4jConfigurer.initLogger(); - - InputStream is = getKylinPropertiesAsInputSteam(); - if (is == null) { - throw new IllegalArgumentException("Failed to load kylin config"); - } - KylinConfig config = new KylinConfig(); - config.reloadKylinConfig(is); - - return config; - } - - private PropertiesConfiguration kylinConfig = new PropertiesConfiguration(); - - private String metadataUrl; - private String storageUrl; - - public CliCommandExecutor getCliCommandExecutor() throws IOException { - CliCommandExecutor exec = new CliCommandExecutor(); - if (getRunAsRemoteCommand()) { - exec.setRunAtRemote(getRemoteHadoopCliHostname(), getRemoteHadoopCliUsername(), getRemoteHadoopCliPassword()); - } - return exec; - } - - // ============================================================================ - - public boolean isHiveReroutingEnabled() { - return Boolean.parseBoolean(getOptional("kylin.route.hive.enabled", "false")); - } - - public String getHiveRerouteUrl() { - return getOptional("kylin.route.hive.url", "jdbc:hive2://"); - } - - public String getHiveRerouteUsername() { - return getOptional("kylin.route.hive.username", ""); - } - - public String getHiveReroutePassword() { - return getOptional("kylin.route.hive.password", ""); - } - - public String getStorageUrl() { - return storageUrl; - } - - /** Use the hive reroute feature instead */ - @Deprecated - public String getHiveUrl() { - return getOptional("hive.url", ""); - } - - /** Use the hive reroute feature instead */ - @Deprecated - public String getHiveUser() { - return getOptional("hive.user", ""); - } - - /** Use the hive reroute feature instead */ - @Deprecated - public String getHivePassword() { - return getOptional("hive.password", ""); - } - - public String getHdfsWorkingDirectory() { - String root = getRequired(KYLIN_HDFS_WORKING_DIR); - if (!root.endsWith("/")) { - root += "/"; - } - return new StringBuffer(root) - .append(StringUtils.replaceChars(getMetadataUrlPrefix(), ':', '-')) - .append("/") - .toString(); - } - - public String getHBaseClusterFs() { - return getOptional(KYLIN_HBASE_CLUSTER_FS, ""); - } - - public String getKylinJobLogDir() { - return getOptional(KYLIN_JOB_LOG_DIR, "/tmp/kylin/logs"); - } - - public String getKylinJobJarPath() { - final String jobJar = getOptional(KYLIN_JOB_JAR); - if (StringUtils.isNotEmpty(jobJar)) { - return jobJar; - } - String kylinHome = getKylinHome(); - if (StringUtils.isEmpty(kylinHome)) { - return ""; - } - return getFileName(kylinHome + File.separator + "lib", JOB_JAR_NAME_PATTERN); - } - - public String getKylinJobMRLibDir() { - return getOptional("kylin.job.mr.lib.dir", ""); - } - - public void overrideKylinJobJarPath(String path) { - logger.info("override " + KYLIN_JOB_JAR + " to " + path); - System.setProperty(KYLIN_JOB_JAR, path); - } - - private static final Pattern COPROCESSOR_JAR_NAME_PATTERN = Pattern.compile("kylin-coprocessor-(.+)\\.jar"); - private static final Pattern JOB_JAR_NAME_PATTERN = Pattern.compile("kylin-job-(.+)\\.jar"); - - public String getCoprocessorLocalJar() { - final String coprocessorJar = getOptional(COPROCESSOR_LOCAL_JAR); - if (StringUtils.isNotEmpty(coprocessorJar)) { - return coprocessorJar; - } - String kylinHome = getKylinHome(); - if (StringUtils.isEmpty(kylinHome)) { - throw new RuntimeException("getCoprocessorLocalJar needs KYLIN_HOME"); - } - return getFileName(kylinHome + File.separator + "lib", COPROCESSOR_JAR_NAME_PATTERN); - } - - private static String getFileName(String homePath, Pattern pattern) { - File home = new File(homePath); - SortedSet<String> files = Sets.newTreeSet(); - if (home.exists() && home.isDirectory()) { - for (File file : home.listFiles()) { - final Matcher matcher = pattern.matcher(file.getName()); - if (matcher.matches()) { - files.add(file.getAbsolutePath()); - } - } - } - if (files.isEmpty()) { - throw new RuntimeException("cannot find " + pattern.toString() + " in " + homePath); - } else { - return files.last(); - } - } - - public void overrideCoprocessorLocalJar(String path) { - logger.info("override " + COPROCESSOR_LOCAL_JAR + " to " + path); - System.setProperty(COPROCESSOR_LOCAL_JAR, path); - } - - public double getDefaultHadoopJobReducerInputMB() { - return Double.parseDouble(getOptional(KYLIN_JOB_MAPREDUCE_DEFAULT_REDUCE_INPUT_MB, "500")); - } - - public double getDefaultHadoopJobReducerCountRatio() { - return Double.parseDouble(getOptional(KYLIN_JOB_MAPREDUCE_DEFAULT_REDUCE_COUNT_RATIO, "1.0")); - } - - public int getHadoopJobMaxReducerNumber() { - return Integer.parseInt(getOptional(KYLIN_JOB_MAPREDUCE_MAX_REDUCER_NUMBER, "5000")); - } - - public boolean getRunAsRemoteCommand() { - return Boolean.parseBoolean(getOptional(KYLIN_JOB_RUN_AS_REMOTE_CMD)); - } - - public String getRemoteHadoopCliHostname() { - return getOptional(KYLIN_JOB_REMOTE_CLI_HOSTNAME); - } - - public String getRemoteHadoopCliUsername() { - return getOptional(KYLIN_JOB_REMOTE_CLI_USERNAME); - } - - public String getRemoteHadoopCliPassword() { - return getOptional(KYLIN_JOB_REMOTE_CLI_PASSWORD); - } - - public String getCliWorkingDir() { - return getOptional(KYLIN_JOB_REMOTE_CLI_WORKING_DIR); - } - - public String getMapReduceCmdExtraArgs() { - return getOptional(KYLIN_JOB_CMD_EXTRA_ARGS); - } - - public boolean getKylinUseKerberosAuth() { - return Boolean.valueOf(getOptional(KYLIN_GET_JOB_STATUS_WITH_KERBEROS, "false")); - } - - public String getOverrideHiveTableLocation(String table) { - return getOptional(HIVE_TABLE_LOCATION_PREFIX + table.toUpperCase()); - } - - public String getTempHDFSDir() { - return getOptional(KYLIN_TMP_HDFS_DIR, "/tmp/kylin"); - } - - public String getYarnStatusCheckUrl() { - return getOptional(KYLIN_JOB_YARN_APP_REST_CHECK_URL, null); - } - - public int getYarnStatusCheckIntervalSeconds() { - return Integer.parseInt(getOptional(KYLIN_JOB_YARN_APP_REST_CHECK_INTERVAL_SECONDS, "60")); - } - - public int getMaxConcurrentJobLimit() { - return Integer.parseInt(getOptional(KYLIN_JOB_CONCURRENT_MAX_LIMIT, "10")); - } - - public String getTimeZone() { - return getOptional(KYLIN_REST_TIMEZONE, "PST"); - } - - public String[] getRestServers() { - return getOptionalStringArray(KYLIN_REST_SERVERS); - } - - public String getAdminDls() { - return getOptional("kylin.job.admin.dls", null); - } - - public long getJobStepTimeout() { - return Long.parseLong(getOptional("kylin.job.step.timeout", String.valueOf(2 * 60 * 60))); - } - - public String getServerMode() { - return this.getOptional("kylin.server.mode", "all"); - } - - public int getCachedDictMaxEntrySize() { - return Integer.parseInt(getOptional("kylin.dict.cache.max.entry", "3000")); - } - - - public int getDictionaryMaxCardinality() { - return Integer.parseInt(getOptional("kylin.dictionary.max.cardinality", "5000000")); - } - - public int getTableSnapshotMaxMB() { - return Integer.parseInt(getOptional("kylin.table.snapshot.max_mb", "300")); - } - - public int getScanThreshold() { - return Integer.parseInt(getOptional("kylin.query.scan.threshold", "10000000")); - } - - public boolean getQueryRunLocalCoprocessor() { - return Boolean.parseBoolean(getOptional("kylin.query.run.local.coprocessor", "false")); - } - - public Long getQueryDurationCacheThreshold() { - return Long.parseLong(this.getOptional("kylin.query.cache.threshold.duration", String.valueOf(2000))); - } - - public Long getQueryScanCountCacheThreshold() { - return Long.parseLong(this.getOptional("kylin.query.cache.threshold.scancount", String.valueOf(10 * 1024))); - } - - public boolean isQuerySecureEnabled() { - return Boolean.parseBoolean(this.getOptional("kylin.query.security.enabled", "true")); - } - - public boolean isQueryCacheEnabled() { - return Boolean.parseBoolean(this.getOptional("kylin.query.cache.enabled", "true")); - } - - public long getQueryMemBudget() { - return Long.parseLong(this.getOptional("kylin.query.mem.budget", String.valueOf(3L * 1024 * 1024 * 1024))); - } - - public int getHBaseKeyValueSize() { - return Integer.parseInt(this.getOptional("kylin.hbase.client.keyvalue.maxsize", "10485760")); - } - - public int getHBaseScanCacheRows() { - return Integer.parseInt(this.getOptional("kylin.hbase.scan.cache_rows", "1024")); - } - - public int getHBaseScanMaxResultSize() { - return Integer.parseInt(this.getOptional("kylin.hbase.scan.max_result_size", "" + (5 * 1024 * 1024))); // 5 MB - } - - public boolean isQueryIgnoreUnknownFunction() { - return Boolean.parseBoolean(this.getOptional("kylin.query.ignore_unknown_function", "false")); - } - - public String getHbaseDefaultCompressionCodec() { - return getOptional(HTABLE_DEFAULT_COMPRESSION_CODEC, ""); - - } - - private String getOptional(String prop) { - final String property = System.getProperty(prop); - return property != null ? property : kylinConfig.getString(prop); - } - - private String[] getOptionalStringArray(String prop) { - final String property = System.getProperty(prop); - if (!StringUtils.isBlank(property)) - return property.split("\\s*,\\s*"); - - return kylinConfig.getStringArray(prop); - } - - private String getOptional(String prop, String dft) { - final String property = System.getProperty(prop); - return property != null ? property : kylinConfig.getString(prop, dft); - } - - private String getRequired(String prop) { - final String property = System.getProperty(prop); - if (property != null) { - return property; - } - String r = kylinConfig.getString(prop); - if (StringUtils.isEmpty(r)) { - throw new IllegalArgumentException("missing '" + prop + "' in conf/kylin_instance.properties"); - } - return r; - } - - void reloadKylinConfig(InputStream is) { - PropertiesConfiguration config = new PropertiesConfiguration(); - try { - config.load(is); - } catch (ConfigurationException e) { - throw new RuntimeException("Cannot load kylin config.", e); - } finally { - try { - is.close(); - } catch (IOException e) { - logger.error("Failed to close inputstream.", e); - } - } - this.kylinConfig = config; - this.metadataUrl = getOptional(KYLIN_METADATA_URL); - this.storageUrl = getOptional(KYLIN_STORAGE_URL); - } - - public void writeProperties(File file) throws IOException { - try { - kylinConfig.save(file); - } catch (ConfigurationException ex) { - throw new IOException("Error writing KylinConfig to " + file, ex); - } - } - - public static String getKylinHome() { - String kylinHome = System.getenv(KYLIN_HOME); - if (StringUtils.isEmpty(kylinHome)) { - logger.warn("KYLIN_HOME was not set"); - return kylinHome; - } - return kylinHome; - } - - public void printProperties() throws IOException { - try { - kylinConfig.save(System.out); - } catch (ConfigurationException ex) { - throw new IOException("Error printing KylinConfig", ex); - } - } - - private static File getKylinProperties() { - String kylinConfHome = System.getProperty(KYLIN_CONF); - if (!StringUtils.isEmpty(kylinConfHome)) { - logger.info("Use KYLIN_CONF=" + kylinConfHome); - return getKylinPropertiesFile(kylinConfHome); - } - - logger.warn("KYLIN_CONF property was not set, will seek KYLIN_HOME env variable"); - - String kylinHome = getKylinHome(); - if (StringUtils.isEmpty(kylinHome)) - throw new RuntimeException("Didn't find KYLIN_CONF or KYLIN_HOME, please set one of them"); - - String path = kylinHome + File.separator + "conf"; - return getKylinPropertiesFile(path); - - } - - public static InputStream getKylinPropertiesAsInputSteam() { - File propFile = getKylinProperties(); - if (propFile == null || !propFile.exists()) { - logger.error("fail to locate kylin.properties"); - throw new RuntimeException("fail to locate kylin.properties"); - } - - File overrideFile = new File(propFile.getParentFile(), propFile.getName() + ".override"); - if (overrideFile.exists()) { - try { - PropertiesConfiguration conf = new PropertiesConfiguration(); - conf.load(propFile); - PropertiesConfiguration override = new PropertiesConfiguration(); - override.load(overrideFile); - conf.copy(override); - ByteArrayOutputStream bout = new ByteArrayOutputStream(); - conf.save(bout); - return new ByteArrayInputStream(bout.toByteArray()); - } catch (ConfigurationException e) { - throw new RuntimeException(e); - } - } - - try { - return new FileInputStream(propFile); - } catch (FileNotFoundException e) { - logger.error("this should not happen"); - throw new RuntimeException(e); - } - - } - - /** - * Check if there is kylin.properties exist - * - * @param path - * @return the properties file - */ - private static File getKylinPropertiesFile(String path) { - if (path == null) { - return null; - } - - return new File(path, KYLIN_CONF_PROPERTIES_FILE); - } - - public String getMetadataUrl() { - return metadataUrl; - } - - public String getMetadataUrlPrefix() { - String hbaseMetadataUrl = getMetadataUrl(); - String defaultPrefix = "kylin_metadata"; - - if (hbaseMetadataUrl.indexOf("@hbase") > 0) { - int cut = hbaseMetadataUrl.indexOf('@'); - String tmp = cut < 0 ? defaultPrefix : hbaseMetadataUrl.substring(0, cut); - return tmp; - } else { - return defaultPrefix; - } - } - - public void setMetadataUrl(String metadataUrl) { - kylinConfig.setProperty(KYLIN_METADATA_URL, metadataUrl); - this.metadataUrl = metadataUrl; - } - - public void setStorageUrl(String storageUrl) { - kylinConfig.setProperty(KYLIN_STORAGE_URL, storageUrl); - this.storageUrl = storageUrl; - } - - public String getHiveDatabaseForIntermediateTable() { - return this.getOptional(HIVE_DATABASE_FOR_INTERMEDIATE_TABLE, "default"); - } - - public void setRunAsRemoteCommand(String v) { - kylinConfig.setProperty(KYLIN_JOB_RUN_AS_REMOTE_CMD, v); - } - - public void setRemoteHadoopCliHostname(String v) { - kylinConfig.setProperty(KYLIN_JOB_REMOTE_CLI_HOSTNAME, v); - } - - public void setRemoteHadoopCliUsername(String v) { - kylinConfig.setProperty(KYLIN_JOB_REMOTE_CLI_USERNAME, v); - } - - public void setRemoteHadoopCliPassword(String v) { - kylinConfig.setProperty(KYLIN_JOB_REMOTE_CLI_PASSWORD, v); - } - - public int getHBaseRegionCountMin() { - return Integer.parseInt(getOptional(HBASE_REGION_COUNT_MIN, "1")); - } - - public int getHBaseRegionCountMax() { - return Integer.parseInt(getOptional(HBASE_REGION_COUNT_MAX, "500")); - } - - public int getHBaseRegionCut(String capacity) { - String cut; - switch (capacity) { - case "SMALL": - cut = getProperty(HBASE_REGION_CUT_SMALL, "10"); - break; - case "MEDIUM": - cut = getProperty(HBASE_REGION_CUT_MEDIUM, "20"); - break; - case "LARGE": - cut = getProperty(HBASE_REGION_CUT_LARGE, "100"); - break; - default: - throw new IllegalArgumentException("Capacity not recognized: " + capacity); - } - - return Integer.valueOf(cut); - } - - public String getProperty(String key, String defaultValue) { - return kylinConfig.getString(key, defaultValue); - } - - /** - * Set a new key:value into the kylin config. - * - * @param key - * @param value - */ - public void setProperty(String key, String value) { - logger.info("Kylin Config was updated with " + key + " : " + value); - kylinConfig.setProperty(key, value); - } - - public String getConfigAsString() throws IOException { - try { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - kylinConfig.save(baos); - String content = baos.toString(); - return content; - } catch (ConfigurationException ex) { - throw new IOException("Error writing KylinConfig to String", ex); - } - } - - public String toString() { - return getMetadataUrl(); - } - -}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java b/common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java deleted file mode 100644 index 6095295..0000000 --- a/common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java +++ /dev/null @@ -1,98 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.kylin.common.debug; - -import java.util.Map; - -/** - */ -public class BackdoorToggles { - - private static final ThreadLocal<Map<String, String>> _backdoorToggles = new ThreadLocal<Map<String, String>>(); - - public static void setToggles(Map<String, String> toggles) { - _backdoorToggles.set(toggles); - } - - public static String getObserverBehavior() { - return getString(DEBUG_TOGGLE_OBSERVER_BEHAVIOR); - } - - public static boolean getDisableFuzzyKey() { - return getBoolean(DEBUG_TOGGLE_DISABLE_FUZZY_KEY); - } - - public static boolean getRunLocalCoprocessor() { - return getBoolean(DEBUG_TOGGLE_LOCAL_COPROCESSOR); - } - - private static String getString(String key) { - Map<String, String> toggles = _backdoorToggles.get(); - if (toggles == null) { - return null; - } else { - return toggles.get(key); - } - } - - private static boolean getBoolean(String key) { - return "true".equals(getString(key)); - } - - public static void cleanToggles() { - _backdoorToggles.remove(); - } - - /** - * set DEBUG_TOGGLE_DISABLE_FUZZY_KEY=true to disable fuzzy key for debug/profile usage - * - * - * - example: - - "backdoorToggles": { - "DEBUG_TOGGLE_DISABLE_FUZZY_KEY": "true" - } - - */ - public final static String DEBUG_TOGGLE_DISABLE_FUZZY_KEY = "DEBUG_TOGGLE_DISABLE_FUZZY_KEY"; - - /** - * set DEBUG_TOGGLE_OBSERVER_BEHAVIOR=SCAN/SCAN_FILTER/SCAN_FILTER_AGGR to control observer behavior for debug/profile usage - * - example: - - "backdoorToggles": { - "DEBUG_TOGGLE_OBSERVER_BEHAVIOR": "SCAN" - } - - */ - public final static String DEBUG_TOGGLE_OBSERVER_BEHAVIOR = "DEBUG_TOGGLE_OBSERVER_BEHAVIOR"; - - /** - * set DEBUG_TOGGLE_LOCAL_COPROCESSOR=true to run coprocessor at client side (not in HBase region server) - * - example: - - "backdoorToggles": { - "DEBUG_TOGGLE_LOCAL_COPROCESSOR": "true" - } - - */ - public final static String DEBUG_TOGGLE_LOCAL_COPROCESSOR = "DEBUG_TOGGLE_LOCAL_COPROCESSOR"; -} http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/common/src/main/java/org/apache/kylin/common/hll/HyperLogLogPlusCounter.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/kylin/common/hll/HyperLogLogPlusCounter.java b/common/src/main/java/org/apache/kylin/common/hll/HyperLogLogPlusCounter.java deleted file mode 100644 index d37d6db..0000000 --- a/common/src/main/java/org/apache/kylin/common/hll/HyperLogLogPlusCounter.java +++ /dev/null @@ -1,340 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -package org.apache.kylin.common.hll; - -import java.io.IOException; -import java.io.Serializable; -import java.nio.ByteBuffer; -import java.nio.charset.Charset; -import java.util.Arrays; - -import org.apache.kylin.common.util.BytesUtil; - -import com.google.common.hash.HashFunction; -import com.google.common.hash.Hashing; - -/** - * About compression, test on HLLC data shows - * - * - LZF compression ratio is around 65%-80%, fast - * - GZIP compression ratio is around 41%-46%, very slow - * - * @author yangli9 - */ -@SuppressWarnings("serial") -public class HyperLogLogPlusCounter implements Serializable, Comparable<HyperLogLogPlusCounter> { - - private final int p; - private final int m; - private final HashFunction hashFunc; - byte[] registers; - - public HyperLogLogPlusCounter() { - this(10); - } - - public HyperLogLogPlusCounter(int p) { - this(p, Hashing.murmur3_128()); - } - - public HyperLogLogPlusCounter(HyperLogLogPlusCounter another) { - this(another.p, another.hashFunc); - merge(another); - } - - /** The larger p is, the more storage (2^p bytes), the better accuracy */ - private HyperLogLogPlusCounter(int p, HashFunction hashFunc) { - this.p = p; - this.m = 1 << p;//(int) Math.pow(2, p); - this.hashFunc = hashFunc; - this.registers = new byte[m]; - } - - public void clear() { - byte zero = (byte) 0; - Arrays.fill(registers, zero); - } - - public void add(int value) { - add(hashFunc.hashInt(value).asLong()); - } - - public void add(String value) { - add(hashFunc.hashString(value, Charset.defaultCharset()).asLong()); - } - - public void add(byte[] value) { - add(hashFunc.hashBytes(value).asLong()); - } - - public void add(byte[] value, int offset, int length) { - add(hashFunc.hashBytes(value, offset, length).asLong()); - } - - protected void add(long hash) { - int bucketMask = m - 1; - int bucket = (int) (hash & bucketMask); - int firstOnePos = Long.numberOfLeadingZeros(hash | bucketMask) + 1; - - if (firstOnePos > registers[bucket]) - registers[bucket] = (byte) firstOnePos; - } - - public void merge(HyperLogLogPlusCounter another) { - assert this.p == another.p; - assert this.hashFunc == another.hashFunc; - - for (int i = 0; i < m; i++) { - if (registers[i] < another.registers[i]) - registers[i] = another.registers[i]; - } - } - - public long getCountEstimate() { - return new HLLCSnapshot(this).getCountEstimate(); - } - - public double getErrorRate() { - return 1.04 / Math.sqrt(m); - } - - private int size() { - int size = 0; - for (int i = 0; i < m; i++) { - if (registers[i] > 0) - size++; - } - return size; - } - - @Override - public String toString() { - return "" + getCountEstimate(); - } - - // ============================================================================ - - // a memory efficient snapshot of HLL registers which can yield count - // estimate later - public static class HLLCSnapshot { - byte p; - double registerSum; - int zeroBuckets; - - public HLLCSnapshot(HyperLogLogPlusCounter hllc) { - p = (byte) hllc.p; - registerSum = 0; - zeroBuckets = 0; - - byte[] registers = hllc.registers; - for (int i = 0; i < hllc.m; i++) { - if (registers[i] == 0) { - registerSum++; - zeroBuckets++; - } else { - registerSum += 1.0 / (1 << registers[i]); - } - } - } - - public long getCountEstimate() { - int m = (int) Math.pow(2, p); - double alpha = 1 / (2 * Math.log(2) * (1 + (3 * Math.log(2) - 1) / m)); - double alphaMM = alpha * m * m; - double estimate = alphaMM / registerSum; - - // small cardinality adjustment - if (zeroBuckets >= m * 0.07) { // (reference presto's HLL impl) - estimate = m * Math.log(m * 1.0 / zeroBuckets); - } else if (HyperLogLogPlusTable.isBiasCorrection(m, estimate)) { - estimate = HyperLogLogPlusTable.biasCorrection(p, estimate); - } - - return Math.round(estimate); - } - } - - // ============================================================================ - - public void writeRegisters(final ByteBuffer out) throws IOException { - - final int indexLen = getRegisterIndexSize(); - int size = size(); - - // decide output scheme -- map (3*size bytes) or array (2^p bytes) - byte scheme; - if (5 + (indexLen + 1) * size < m) // 5 is max len of vint - scheme = 0; // map - else - scheme = 1; // array - out.put(scheme); - - if (scheme == 0) { // map scheme - BytesUtil.writeVInt(size, out); - for (int i = 0; i < m; i++) { - if (registers[i] > 0) { - writeUnsigned(i, indexLen, out); - out.put(registers[i]); - } - } - } else if (scheme == 1) { // array scheme - out.put(registers); - } else - throw new IllegalStateException(); - } - - public void readRegisters(ByteBuffer in) throws IOException { - byte scheme = in.get(); - - if (scheme == 0) { // map scheme - clear(); - int size = BytesUtil.readVInt(in); - if (size > m) - throw new IllegalArgumentException("register size (" + size + ") cannot be larger than m (" + m + ")"); - int indexLen = getRegisterIndexSize(); - for (int i = 0; i < size; i++) { - int key = readUnsigned(in, indexLen); - registers[key] = in.get(); - } - } else if (scheme == 1) { // array scheme - in.get(registers); - } else - throw new IllegalStateException(); - } - - public int peekLength(ByteBuffer in) { - int mark = in.position(); - int len; - - byte scheme = in.get(); - if (scheme == 0) { // map scheme - int size = BytesUtil.readVInt(in); - int indexLen = getRegisterIndexSize(); - len = in.position() - mark + (indexLen + 1) * size; - } else { - len = in.position() - mark + m; - } - - in.position(mark); - return len; - } - - public int maxLength() { - return 1 + m; - } - - public void writeRegistersArray(final ByteBuffer out) { - out.put(this.registers); - } - - public void readRegistersArray(ByteBuffer in) { - in.get(registers, 0, m); - } - - private int getRegisterIndexSize() { - return (p - 1) / 8 + 1; // 2 when p=16, 3 when p=17 - } - - @Override - public int hashCode() { - final int prime = 31; - int result = 1; - result = prime * result + ((hashFunc == null) ? 0 : hashFunc.hashCode()); - result = prime * result + p; - result = prime * result + Arrays.hashCode(registers); - return result; - } - - @Override - public boolean equals(Object obj) { - if (this == obj) - return true; - if (obj == null) - return false; - if (getClass() != obj.getClass()) - return false; - HyperLogLogPlusCounter other = (HyperLogLogPlusCounter) obj; - if (hashFunc == null) { - if (other.hashFunc != null) - return false; - } else if (!hashFunc.equals(other.hashFunc)) - return false; - if (p != other.p) - return false; - if (!Arrays.equals(registers, other.registers)) - return false; - return true; - } - - @Override - public int compareTo(HyperLogLogPlusCounter o) { - if (o == null) - return 1; - - long e1 = this.getCountEstimate(); - long e2 = o.getCountEstimate(); - - if (e1 == e2) - return 0; - else if (e1 > e2) - return 1; - else - return -1; - } - - public static void main(String[] args) throws IOException { - dumpErrorRates(); - } - - static void dumpErrorRates() { - for (int p = 10; p <= 18; p++) { - double rate = new HyperLogLogPlusCounter(p).getErrorRate(); - double er = Math.round(rate * 10000) / 100D; - double er2 = Math.round(rate * 2 * 10000) / 100D; - double er3 = Math.round(rate * 3 * 10000) / 100D; - long size = Math.round(Math.pow(2, p)); - System.out.println("HLLC" + p + ",\t" + size + " bytes,\t68% err<" + er + "%" + ",\t95% err<" + er2 + "%" + ",\t99.7% err<" + er3 + "%"); - } - } - - /** - * - * @param num - * @param size - * @param out - */ - public static void writeUnsigned(int num, int size, ByteBuffer out) { - for (int i = 0; i < size; i++) { - out.put((byte) num); - num >>>= 8; - } - } - - public static int readUnsigned(ByteBuffer in, int size) { - int integer = 0; - int mask = 0xff; - int shift = 0; - for (int i = 0; i < size; i++) { - integer |= (in.get() << shift) & mask; - mask = mask << 8; - shift += 8; - } - return integer; - } -}
