http://git-wip-us.apache.org/repos/asf/hive/blob/187eb760/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java ---------------------------------------------------------------------- diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 30eefa6..d32f1e5 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -272,6 +272,7 @@ public class HiveConf extends Configuration { HiveConf.ConfVars.HIVE_TXN_HEARTBEAT_THREADPOOL_SIZE, HiveConf.ConfVars.HIVE_TXN_MAX_OPEN_BATCH, HiveConf.ConfVars.HIVE_TXN_RETRYABLE_SQLEX_REGEX, + HiveConf.ConfVars.HIVE_METASTORE_STATS_NDV_TUNER, HiveConf.ConfVars.HIVE_METASTORE_STATS_NDV_DENSITY_FUNCTION, HiveConf.ConfVars.METASTORE_AGGREGATE_STATS_CACHE_ENABLED, HiveConf.ConfVars.METASTORE_AGGREGATE_STATS_CACHE_SIZE, @@ -550,6 +551,13 @@ public class HiveConf extends Configuration { "If not set, defaults to the codec extension for text files (e.g. \".gz\"), or no extension otherwise."), HIVE_IN_TEST("hive.in.test", false, "internal usage only, true in test mode", true), + HIVE_IN_TEST_SHORT_LOGS("hive.in.test.short.logs", false, + "internal usage only, used only in test mode. If set true, when requesting the " + + "operation logs the short version (generated by LogDivertAppenderForTest) will be " + + "returned"), + HIVE_IN_TEST_REMOVE_LOGS("hive.in.test.remove.logs", true, + "internal usage only, used only in test mode. If set false, the operation logs, and the " + + "operation log directory will not be removed, so they can be found after the test runs."), HIVE_IN_TEZ_TEST("hive.in.tez.test", false, "internal use only, true when in testing tez", true), @@ -736,9 +744,16 @@ public class HiveConf extends Configuration { "Defaults to all permissions for the hiveserver2/metastore process user."), METASTORE_CACHE_PINOBJTYPES("hive.metastore.cache.pinobjtypes", "Table,StorageDescriptor,SerDeInfo,Partition,Database,Type,FieldSchema,Order", "List of comma separated metastore object types that should be pinned in the cache"), - METASTORE_CONNECTION_POOLING_TYPE("datanucleus.connectionPoolingType", "BONECP", new StringSet("BONECP", "DBCP", + METASTORE_CONNECTION_POOLING_TYPE("datanucleus.connectionPoolingType", "HikariCP", new StringSet("BONECP", "DBCP", "HikariCP", "NONE"), "Specify connection pool library for datanucleus"), + METASTORE_CONNECTION_POOLING_MAX_CONNECTIONS("datanucleus.connectionPool.maxPoolSize", 10, + "Specify the maximum number of connections in the connection pool. Note: The configured size will be used by\n" + + "2 connection pools (TxnHandler and ObjectStore). When configuring the max connection pool size, it is\n" + + "recommended to take into account the number of metastore instances and the number of HiveServer2 instances\n" + + "configured with embedded metastore. To get optimal performance, set config to meet the following condition\n"+ + "(2 * pool_size * metastore_instances + 2 * pool_size * HS2_instances_with_embedded_metastore) = \n" + + "(2 * physical_core_count + hard_disk_count)."), // Workaround for DN bug on Postgres: // http://www.datanucleus.org/servlet/forum/viewthread_thread,7985_offset METASTORE_DATANUCLEUS_INIT_COL_INFO("datanucleus.rdbms.initializeColumnInfo", "NONE", @@ -887,6 +902,12 @@ public class HiveConf extends Configuration { METASTORE_RAW_STORE_IMPL("hive.metastore.rawstore.impl", "org.apache.hadoop.hive.metastore.ObjectStore", "Name of the class that implements org.apache.hadoop.hive.metastore.rawstore interface. \n" + "This class is used to store and retrieval of raw metadata objects such as table, database"), + METASTORE_CACHED_RAW_STORE_IMPL("hive.metastore.cached.rawstore.impl", "org.apache.hadoop.hive.metastore.ObjectStore", + "Name of the wrapped RawStore class"), + METASTORE_CACHED_RAW_STORE_CACHE_UPDATE_FREQUENCY( + "hive.metastore.cached.rawstore.cache.update.frequency", "60", new TimeValidator( + TimeUnit.SECONDS), + "The time after which metastore cache is updated from metastore DB."), METASTORE_TXN_STORE_IMPL("hive.metastore.txn.store.impl", "org.apache.hadoop.hive.metastore.txn.CompactionTxnHandler", "Name of class that implements org.apache.hadoop.hive.metastore.txn.TxnStore. This " + @@ -1688,6 +1709,10 @@ public class HiveConf extends Configuration { HIVE_STATS_NDV_ERROR("hive.stats.ndv.error", (float)20.0, "Standard error expressed in percentage. Provides a tradeoff between accuracy and compute cost. \n" + "A lower value for error indicates higher accuracy and a higher compute cost."), + HIVE_METASTORE_STATS_NDV_TUNER("hive.metastore.stats.ndv.tuner", (float)0.0, + "Provides a tunable parameter between the lower bound and the higher bound of ndv for aggregate ndv across all the partitions. \n" + + "The lower bound is equal to the maximum of ndv of all the partitions. The higher bound is equal to the sum of ndv of all the partitions.\n" + + "Its value should be between 0.0 (i.e., choose lower bound) and 1.0 (i.e., choose higher bound)"), HIVE_METASTORE_STATS_NDV_DENSITY_FUNCTION("hive.metastore.stats.ndv.densityfunction", false, "Whether to use density function to estimate the NDV for the whole table based on the NDV of partitions"), HIVE_STATS_KEY_PREFIX("hive.stats.key.prefix", "", "", true), // internal usage only @@ -1736,6 +1761,10 @@ public class HiveConf extends Configuration { "uses column statistics to estimate the number of rows flowing out of it and hence the data size.\n" + "In the absence of column statistics, this factor determines the amount of rows that flows out\n" + "of JOIN operator."), + HIVE_STATS_CORRELATED_MULTI_KEY_JOINS("hive.stats.correlated.multi.key.joins", false, + "When estimating output rows for a join involving multiple columns, the default behavior assumes" + + "the columns are independent. Setting this flag to true will cause the estimator to assume" + + "the columns are correlated."), // in the absence of uncompressed/raw data size, total file size will be used for statistics // annotation. But the file may be compressed, encoded and serialized which may be lesser in size // than the actual uncompressed/raw data size. This factor will be multiplied to file size to estimate @@ -1767,6 +1796,9 @@ public class HiveConf extends Configuration { HIVE_LOCK_MAPRED_ONLY("hive.lock.mapred.only.operation", false, "This param is to control whether or not only do lock on queries\n" + "that need to execute at least one mapred job."), + HIVE_LOCK_QUERY_STRING_MAX_LENGTH("hive.lock.query.string.max.length", 1000000, + "The maximum length of the query string to store in the lock.\n" + + "The default value is 1000000, since the data limit of a znode is 1MB"), // Zookeeper related configs HIVE_ZOOKEEPER_QUORUM("hive.zookeeper.quorum", "", @@ -1999,6 +2031,7 @@ public class HiveConf extends Configuration { HIVE_DRUID_PASSIVE_WAIT_TIME("hive.druid.passiveWaitTimeMs", 30000, "Wait time in ms default to 30 seconds." ), + HIVE_DRUID_BITMAP_FACTORY_TYPE("hive.druid.bitmap.type", "roaring", new PatternSet("roaring", "concise"), "Coding algorithm use to encode the bitmaps"), // For HBase storage handler HIVE_HBASE_WAL_ENABLED("hive.hbase.wal.enabled", true, "Whether writes to HBase should be forced to the write-ahead log. \n" + @@ -2140,7 +2173,7 @@ public class HiveConf extends Configuration { "When true the HDFS location stored in the index file will be ignored at runtime.\n" + "If the data got moved or the name of the cluster got changed, the index data should still be usable."), - HIVE_EXIM_URI_SCHEME_WL("hive.exim.uri.scheme.whitelist", "hdfs,pfile,file", + HIVE_EXIM_URI_SCHEME_WL("hive.exim.uri.scheme.whitelist", "hdfs,pfile,file,s3,s3a", "A comma separated list of acceptable URI schemes for import and export."), // temporary variable for testing. This is added just to turn off this feature in case of a bug in // deployment. It has not been documented in hive-default.xml intentionally, this should be removed @@ -2192,7 +2225,10 @@ public class HiveConf extends Configuration { "When enabled, will log EXPLAIN EXTENDED output for the query at INFO log4j log level."), HIVE_EXPLAIN_USER("hive.explain.user", true, "Whether to show explain result at user level.\n" + - "When enabled, will log EXPLAIN output for the query at user level."), + "When enabled, will log EXPLAIN output for the query at user level. Tez only."), + HIVE_SPARK_EXPLAIN_USER("hive.spark.explain.user", false, + "Whether to show explain result at user level.\n" + + "When enabled, will log EXPLAIN output for the query at user level. Spark only."), // prefix used to auto generated column aliases (this should be started with '_') HIVE_AUTOGEN_COLUMNALIAS_PREFIX_LABEL("hive.autogen.columnalias.prefix.label", "_c", @@ -2208,19 +2244,29 @@ public class HiveConf extends Configuration { "org.apache.hadoop.hive.common.metrics.metrics2.CodahaleMetrics", "org.apache.hadoop.hive.common.metrics.LegacyMetrics"), "Hive metrics subsystem implementation class."), - HIVE_METRICS_REPORTER("hive.service.metrics.reporter", "JSON_FILE, JMX", - "Reporter type for metric class org.apache.hadoop.hive.common.metrics.metrics2.CodahaleMetrics, " + - "comma separated list of JMX, CONSOLE, JSON_FILE, HADOOP2"), + HIVE_CODAHALE_METRICS_REPORTER_CLASSES("hive.service.metrics.codahale.reporter.classes", + "org.apache.hadoop.hive.common.metrics.metrics2.JsonFileMetricsReporter, " + + "org.apache.hadoop.hive.common.metrics.metrics2.JmxMetricsReporter", + "Comma separated list of reporter implementation classes for metric class " + + "org.apache.hadoop.hive.common.metrics.metrics2.CodahaleMetrics. Overrides " + + "HIVE_METRICS_REPORTER conf if present"), + @Deprecated + HIVE_METRICS_REPORTER("hive.service.metrics.reporter", "", + "Reporter implementations for metric class " + + "org.apache.hadoop.hive.common.metrics.metrics2.CodahaleMetrics;" + + "Deprecated, use HIVE_CODAHALE_METRICS_REPORTER_CLASSES instead. This configuraiton will be" + + " overridden by HIVE_CODAHALE_METRICS_REPORTER_CLASSES if present. " + + "Comma separated list of JMX, CONSOLE, JSON_FILE, HADOOP2"), HIVE_METRICS_JSON_FILE_LOCATION("hive.service.metrics.file.location", "/tmp/report.json", "For metric class org.apache.hadoop.hive.common.metrics.metrics2.CodahaleMetrics JSON_FILE reporter, the location of local JSON metrics file. " + "This file will get overwritten at every interval."), - HIVE_METRICS_JSON_FILE_INTERVAL("hive.service.metrics.file.frequency", "5s", + HIVE_METRICS_JSON_FILE_INTERVAL("hive.service.metrics.file.frequency", "5000ms", new TimeValidator(TimeUnit.MILLISECONDS), - "For metric class org.apache.hadoop.hive.common.metrics.metrics2.CodahaleMetrics JSON_FILE reporter, " + + "For metric class org.apache.hadoop.hive.common.metrics.metrics2.JsonFileMetricsReporter, " + "the frequency of updating JSON metrics file."), HIVE_METRICS_HADOOP2_INTERVAL("hive.service.metrics.hadoop2.frequency", "30s", new TimeValidator(TimeUnit.SECONDS), - "For metric class org.apache.hadoop.hive.common.metrics.metrics2.CodahaleMetrics HADOOP2 reporter, " + + "For metric class org.apache.hadoop.hive.common.metrics.metrics2.Metrics2Reporter, " + "the frequency of updating the HADOOP2 metrics system."), HIVE_METRICS_HADOOP2_COMPONENT_NAME("hive.service.metrics.hadoop2.component", "hive", @@ -2237,10 +2283,6 @@ public class HiveConf extends Configuration { HIVE_INSERT_INTO_MULTILEVEL_DIRS("hive.insert.into.multilevel.dirs", false, "Where to insert into multilevel directories like\n" + "\"insert directory '/HIVEFT25686/chinna/' from table\""), - HIVE_WAREHOUSE_SUBDIR_INHERIT_PERMS("hive.warehouse.subdir.inherit.perms", true, - "Set this to false if the table directories should be created\n" + - "with the permissions derived from dfs umask instead of\n" + - "inheriting the permission of the warehouse or database directory."), HIVE_INSERT_INTO_EXTERNAL_TABLES("hive.insert.into.external.tables", true, "whether insert into external tables is allowed"), HIVE_TEMPORARY_TABLE_STORAGE( @@ -2249,7 +2291,10 @@ public class HiveConf extends Configuration { "Choices between memory, ssd and default"), HIVE_QUERY_LIFETIME_HOOKS("hive.query.lifetime.hooks", "", "A comma separated list of hooks which implement QueryLifeTimeHook. These will be triggered" + - " before/after query compilation and before/after query execution, in the order specified"), + " before/after query compilation and before/after query execution, in the order specified." + + "Implementations of QueryLifeTimeHookWithParseHooks can also be specified in this list. If they are" + + "specified then they will be invoked in the same places as QueryLifeTimeHooks and will be invoked during pre " + + "and post query parsing"), HIVE_DRIVER_RUN_HOOKS("hive.exec.driver.run.hooks", "", "A comma separated list of hooks which implement HiveDriverRunHook. Will be run at the beginning " + "and end of Driver.run, these will be run in the order specified."), @@ -2758,9 +2803,9 @@ public class HiveConf extends Configuration { HIVE_VECTORIZATION_USE_VECTORIZED_INPUT_FILE_FORMAT("hive.vectorized.use.vectorized.input.format", true, "This flag should be set to true to enable vectorizing with vectorized input file format capable SerDe.\n" + "The default value is true."), - HIVE_VECTORIZATION_USE_VECTOR_DESERIALIZE("hive.vectorized.use.vector.serde.deserialize", false, + HIVE_VECTORIZATION_USE_VECTOR_DESERIALIZE("hive.vectorized.use.vector.serde.deserialize", true, "This flag should be set to true to enable vectorizing rows using vector deserialize.\n" + - "The default value is false."), + "The default value is true."), HIVE_VECTORIZATION_USE_ROW_DESERIALIZE("hive.vectorized.use.row.serde.deserialize", false, "This flag should be set to true to enable vectorizing using row deserialize.\n" + "The default value is false."), @@ -2829,6 +2874,9 @@ public class HiveConf extends Configuration { "Turn on Tez' auto reducer parallelism feature. When enabled, Hive will still estimate data sizes\n" + "and set parallelism estimates. Tez will sample source vertices' output sizes and adjust the estimates at runtime as\n" + "necessary."), + TEZ_LLAP_MIN_REDUCER_PER_EXECUTOR("hive.tez.llap.min.reducer.per.executor", 0.95f, + "If above 0, the min number of reducers for auto-parallelism for LLAP scheduling will\n" + + "be set to this fraction of the number of executors."), TEZ_MAX_PARTITION_FACTOR("hive.tez.max.partition.factor", 2f, "When auto reducer parallelism is enabled this factor will be used to over-partition data in shuffle edges."), TEZ_MIN_PARTITION_FACTOR("hive.tez.min.partition.factor", 0.25f, @@ -2856,8 +2904,18 @@ public class HiveConf extends Configuration { TEZ_DYNAMIC_SEMIJOIN_REDUCTION("hive.tez.dynamic.semijoin.reduction", true, "When dynamic semijoin is enabled, shuffle joins will perform a leaky semijoin before shuffle. This " + "requires hive.tez.dynamic.partition.pruning to be enabled."), + TEZ_MIN_BLOOM_FILTER_ENTRIES("hive.tez.min.bloom.filter.entries", 1000000L, + "Bloom filter should be of at min certain size to be effective"), TEZ_MAX_BLOOM_FILTER_ENTRIES("hive.tez.max.bloom.filter.entries", 100000000L, "Bloom filter should be of at max certain size to be effective"), + TEZ_BLOOM_FILTER_FACTOR("hive.tez.bloom.filter.factor", (float) 2.0, + "Bloom filter should be a multiple of this factor with nDV"), + TEZ_BIGTABLE_MIN_SIZE_SEMIJOIN_REDUCTION("hive.tez.bigtable.minsize.semijoin.reduction", 100000000L, + "Big table for runtime filteting should be of atleast this size"), + TEZ_DYNAMIC_SEMIJOIN_REDUCTION_THRESHOLD("hive.tez.dynamic.semijoin.reduction.threshold", (float) 0.50, + "Only perform semijoin optimization if the estimated benefit at or above this fraction of the target table"), + TEZ_DYNAMIC_SEMIJOIN_REDUCTION_HINT_ONLY("hive.tez.dynamic.semijoin.reduction.hint.only", false, + "When true, only enforce semijoin when a hint is provided"), TEZ_SMB_NUMBER_WAVES( "hive.tez.smb.number.waves", (float) 0.5, @@ -2895,13 +2953,19 @@ public class HiveConf extends Configuration { "LLAP IO memory usage; 'cache' (the default) uses data and metadata cache with a\n" + "custom off-heap allocator, 'none' doesn't use either (this mode may result in\n" + "significant performance degradation)"), - LLAP_ALLOCATOR_MIN_ALLOC("hive.llap.io.allocator.alloc.min", "16Kb", new SizeValidator(), + LLAP_ALLOCATOR_MIN_ALLOC("hive.llap.io.allocator.alloc.min", "256Kb", new SizeValidator(), "Minimum allocation possible from LLAP buddy allocator. Allocations below that are\n" + "padded to minimum allocation. For ORC, should generally be the same as the expected\n" + "compression buffer size, or next lowest power of 2. Must be a power of 2."), LLAP_ALLOCATOR_MAX_ALLOC("hive.llap.io.allocator.alloc.max", "16Mb", new SizeValidator(), "Maximum allocation possible from LLAP buddy allocator. For ORC, should be as large as\n" + "the largest expected ORC compression buffer size. Must be a power of 2."), + @Deprecated + LLAP_IO_METADATA_FRACTION("hive.llap.io.metadata.fraction", 0.1f, + "Temporary setting for on-heap metadata cache fraction of xmx, set to avoid potential\n" + + "heap problems on very large datasets when on-heap metadata cache takes over\n" + + "everything. -1 managed metadata and data together (which is more flexible). This\n" + + "setting will be removed (in effect become -1) once ORC metadata cache is moved off-heap."), LLAP_ALLOCATOR_ARENA_COUNT("hive.llap.io.allocator.arena.count", 8, "Arena count for LLAP low-level cache; cache will be allocated in the steps of\n" + "(size/arena_count) bytes. This size must be <= 1Gb and >= max allocation; if it is\n" + @@ -3095,6 +3159,19 @@ public class HiveConf extends Configuration { LLAP_DAEMON_NUM_EXECUTORS("hive.llap.daemon.num.executors", 4, "Number of executors to use in LLAP daemon; essentially, the number of tasks that can be\n" + "executed in parallel.", "llap.daemon.num.executors"), + LLAP_MAPJOIN_MEMORY_OVERSUBSCRIBE_FACTOR("hive.llap.mapjoin.memory.oversubscribe.factor", 0.2f, + "Fraction of memory from hive.auto.convert.join.noconditionaltask.size that can be over subscribed\n" + + "by queries running in LLAP mode. This factor has to be from 0.0 to 1.0. Default is 20% over subscription.\n"), + LLAP_MEMORY_OVERSUBSCRIPTION_MAX_EXECUTORS_PER_QUERY("hive.llap.memory.oversubscription.max.executors.per.query", 3, + "Used along with hive.llap.mapjoin.memory.oversubscribe.factor to limit the number of executors from\n" + + "which memory for mapjoin can be borrowed. Default 3 (from 3 other executors\n" + + "hive.llap.mapjoin.memory.oversubscribe.factor amount of memory can be borrowed based on which mapjoin\n" + + "conversion decision will be made). This is only an upper bound. Lower bound is determined by number of\n" + + "executors and configured max concurrency."), + LLAP_MAPJOIN_MEMORY_MONITOR_CHECK_INTERVAL("hive.llap.mapjoin.memory.monitor.check.interval", 100000L, + "Check memory usage of mapjoin hash tables after every interval of this many rows. If map join hash table\n" + + "memory usage exceeds (hive.auto.convert.join.noconditionaltask.size * hive.hash.table.inflation.factor)\n" + + "when running in LLAP, tasks will get killed and not retried. Set the value to 0 to disable this feature."), LLAP_DAEMON_AM_REPORTER_MAX_THREADS("hive.llap.daemon.am-reporter.max.threads", 4, "Maximum number of threads to be used for AM reporter. If this is lower than number of\n" + "executors in llap daemon, it would be set to number of executors at runtime.", @@ -3230,13 +3307,14 @@ public class HiveConf extends Configuration { "logger used for llap-daemons."), SPARK_USE_OP_STATS("hive.spark.use.op.stats", true, - "Whether to use operator stats to determine reducer parallelism for Hive on Spark. " - + "If this is false, Hive will use source table stats to determine reducer " - + "parallelism for all first level reduce tasks, and the maximum reducer parallelism " - + "from all parents for all the rest (second level and onward) reducer tasks."), - SPARK_USE_FILE_SIZE_FOR_MAPJOIN("hive.spark.use.file.size.for.mapjoin", false, - "If this is set to true, mapjoin optimization in Hive/Spark will use source file sizes associated " - + "with TableScan operator on the root of operator tree, instead of using operator statistics."), + "Whether to use operator stats to determine reducer parallelism for Hive on Spark.\n" + + "If this is false, Hive will use source table stats to determine reducer\n" + + "parallelism for all first level reduce tasks, and the maximum reducer parallelism\n" + + "from all parents for all the rest (second level and onward) reducer tasks."), + SPARK_USE_TS_STATS_FOR_MAPJOIN("hive.spark.use.ts.stats.for.mapjoin", false, + "If this is set to true, mapjoin optimization in Hive/Spark will use statistics from\n" + + "TableScan operators at the root of operator tree, instead of parent ReduceSink\n" + + "operators of the Join operator."), SPARK_CLIENT_FUTURE_TIMEOUT("hive.spark.client.future.timeout", "60s", new TimeValidator(TimeUnit.SECONDS), "Timeout for requests from Hive client to remote Spark driver."), @@ -3279,6 +3357,8 @@ public class HiveConf extends Configuration { "hive.spark.use.groupby.shuffle", true, "Spark groupByKey transformation has better performance but uses unbounded memory." + "Turn this off when there is a memory issue."), + SPARK_JOB_MAX_TASKS("hive.spark.job.max.tasks", -1, "The maximum number of tasks a Spark job may have.\n" + + "If a Spark job contains more tasks than the maximum, it will be cancelled. A value of -1 means no limit."), NWAYJOINREORDER("hive.reorder.nway.joins", true, "Runs reordering of tables within single n-way join (i.e.: picks streamtable)"), HIVE_MERGE_NWAY_JOINS("hive.merge.nway.joins", true, @@ -3681,6 +3761,9 @@ public class HiveConf extends Configuration { } } else if (name.startsWith("hive.spark")) { // Remote Spark Context property. result = true; + } else if (name.equals("mapreduce.job.queuename")) { + // a special property starting with mapreduce that we would also like to effect if it changes + result = true; } return result;
http://git-wip-us.apache.org/repos/asf/hive/blob/187eb760/common/src/java/org/apache/hadoop/hive/conf/HiveConfUtil.java ---------------------------------------------------------------------- diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConfUtil.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConfUtil.java index 9ba08e5..dc02803 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConfUtil.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConfUtil.java @@ -94,11 +94,22 @@ public class HiveConfUtil { public static void stripConfigurations(Configuration conf, Set<String> hiddenSet) { for (String name : hiddenSet) { if (conf.get(name) != null) { - conf.set(name, ""); + conf.set(name, StringUtils.EMPTY); } } } + /** + * Searches the given configuration object and replaces all the configuration values for keys + * defined hive.conf.hidden.list by empty String + * + * @param conf - Configuration object which needs to be modified to remove sensitive keys + */ + public static void stripConfigurations(Configuration conf) { + Set<String> hiddenSet = getHiddenSet(conf); + stripConfigurations(conf, hiddenSet); + } + public static void dumpConfig(Configuration originalConf, StringBuilder sb) { Set<String> hiddenSet = getHiddenSet(originalConf); sb.append("Values omitted for security reason if present: ").append(hiddenSet).append("\n"); http://git-wip-us.apache.org/repos/asf/hive/blob/187eb760/common/src/java/org/apache/hadoop/hive/ql/log/PerfLogger.java ---------------------------------------------------------------------- diff --git a/common/src/java/org/apache/hadoop/hive/ql/log/PerfLogger.java b/common/src/java/org/apache/hadoop/hive/ql/log/PerfLogger.java index 7f3c8b3..008b3b0 100644 --- a/common/src/java/org/apache/hadoop/hive/ql/log/PerfLogger.java +++ b/common/src/java/org/apache/hadoop/hive/ql/log/PerfLogger.java @@ -39,6 +39,7 @@ import java.util.Map; public class PerfLogger { public static final String ACQUIRE_READ_WRITE_LOCKS = "acquireReadWriteLocks"; public static final String COMPILE = "compile"; + public static final String WAIT_COMPILE = "waitCompile"; public static final String PARSE = "parse"; public static final String ANALYZE = "semanticAnalyze"; public static final String OPTIMIZER = "optimizer"; http://git-wip-us.apache.org/repos/asf/hive/blob/187eb760/common/src/java/org/apache/hive/http/ConfServlet.java ---------------------------------------------------------------------- diff --git a/common/src/java/org/apache/hive/http/ConfServlet.java b/common/src/java/org/apache/hive/http/ConfServlet.java index 253df4f..856a5d2 100644 --- a/common/src/java/org/apache/hive/http/ConfServlet.java +++ b/common/src/java/org/apache/hive/http/ConfServlet.java @@ -26,6 +26,7 @@ import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.conf.HiveConfUtil; /** * A servlet to print out the running configuration data. @@ -81,11 +82,14 @@ public class ConfServlet extends HttpServlet { * Guts of the servlet - extracted for easy testing. */ static void writeResponse(Configuration conf, Writer out, String format) - throws IOException, BadFormatException { + throws IOException, BadFormatException { + //redact the sensitive information from the configuration values + Configuration hconf = new Configuration(conf); + HiveConfUtil.stripConfigurations(hconf); if (FORMAT_JSON.equals(format)) { - Configuration.dumpConfiguration(conf, out); + Configuration.dumpConfiguration(hconf, out); } else if (FORMAT_XML.equals(format)) { - conf.writeXml(out); + hconf.writeXml(out); } else { throw new BadFormatException("Bad format: " + format); } http://git-wip-us.apache.org/repos/asf/hive/blob/187eb760/common/src/java/org/apache/hive/http/HttpServer.java ---------------------------------------------------------------------- diff --git a/common/src/java/org/apache/hive/http/HttpServer.java b/common/src/java/org/apache/hive/http/HttpServer.java index db5650d..fd3d457 100644 --- a/common/src/java/org/apache/hive/http/HttpServer.java +++ b/common/src/java/org/apache/hive/http/HttpServer.java @@ -42,7 +42,6 @@ import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.authentication.server.AuthenticationFilter; import org.apache.hadoop.security.authorize.AccessControlList; -import org.apache.hadoop.util.Shell; import org.apache.hadoop.hive.common.classification.InterfaceAudience; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.core.Appender; @@ -54,11 +53,13 @@ import org.apache.logging.log4j.core.appender.OutputStreamManager; import org.eclipse.jetty.rewrite.handler.RewriteHandler; import org.eclipse.jetty.rewrite.handler.RewriteRegexRule; import org.eclipse.jetty.server.Connector; +import org.eclipse.jetty.server.HttpConfiguration; +import org.eclipse.jetty.server.HttpConnectionFactory; +import org.eclipse.jetty.server.LowResourceMonitor; import org.eclipse.jetty.server.Server; import org.eclipse.jetty.server.handler.ContextHandler.Context; import org.eclipse.jetty.server.handler.ContextHandlerCollection; -import org.eclipse.jetty.server.nio.SelectChannelConnector; -import org.eclipse.jetty.server.ssl.SslSelectChannelConnector; +import org.eclipse.jetty.server.ServerConnector; import org.eclipse.jetty.servlet.DefaultServlet; import org.eclipse.jetty.servlet.FilterHolder; import org.eclipse.jetty.servlet.FilterMapping; @@ -85,9 +86,9 @@ public class HttpServer { public static final String ADMINS_ACL = "admins.acl"; private final String name; - private final String appDir; - private final WebAppContext webAppContext; - private final Server webServer; + private String appDir; + private WebAppContext webAppContext; + private Server webServer; /** * Create a status server on the given port. @@ -95,16 +96,7 @@ public class HttpServer { private HttpServer(final Builder b) throws IOException { this.name = b.name; - webServer = new Server(); - appDir = getWebAppsPath(b.name); - webAppContext = createWebAppContext(b); - - if (b.useSPNEGO) { - // Secure the web server with kerberos - setupSpnegoFilter(b); - } - - initializeWebServer(b); + createWebServer(b); } public static class Builder { @@ -219,7 +211,7 @@ public class HttpServer { } public int getPort() { - return webServer.getConnectors()[0].getLocalPort(); + return ((ServerConnector)(webServer.getConnectors()[0])).getLocalPort(); } /** @@ -345,9 +337,14 @@ public class HttpServer { * Create a channel connector for "http/https" requests */ Connector createChannelConnector(int queueSize, Builder b) { - SelectChannelConnector connector; + ServerConnector connector; + + final HttpConfiguration conf = new HttpConfiguration(); + conf.setRequestHeaderSize(1024*64); + final HttpConnectionFactory http = new HttpConnectionFactory(conf); + if (!b.useSSL) { - connector = new SelectChannelConnector(); + connector = new ServerConnector(webServer, http); } else { SslContextFactory sslContextFactory = new SslContextFactory(); sslContextFactory.setKeyStorePath(b.keyStorePath); @@ -357,15 +354,13 @@ public class HttpServer { sslContextFactory.addExcludeProtocols(excludedSSLProtocols.toArray( new String[excludedSSLProtocols.size()])); sslContextFactory.setKeyStorePassword(b.keyStorePassword); - connector = new SslSelectChannelConnector(sslContextFactory); + connector = new ServerConnector(webServer, sslContextFactory, http); } - connector.setLowResourcesMaxIdleTime(10000); connector.setAcceptQueueSize(queueSize); - connector.setResolveNames(false); - connector.setUseDirectBuffers(false); - connector.setRequestHeaderSize(1024*64); connector.setReuseAddress(true); + connector.setHost(b.host); + connector.setPort(b.port); return connector; } @@ -378,7 +373,7 @@ public class HttpServer { } } - void initializeWebServer(Builder b) { + private void createWebServer(final Builder b) throws IOException { // Create the thread pool for the web server to handle HTTP requests QueuedThreadPool threadPool = new QueuedThreadPool(); if (b.maxThreads > 0) { @@ -386,12 +381,26 @@ public class HttpServer { } threadPool.setDaemon(true); threadPool.setName(b.name + "-web"); - webServer.setThreadPool(threadPool); - // Create the channel connector for the web server - Connector connector = createChannelConnector(threadPool.getMaxThreads(), b); - connector.setHost(b.host); - connector.setPort(b.port); + this.webServer = new Server(threadPool); + this.appDir = getWebAppsPath(b.name); + this.webAppContext = createWebAppContext(b); + + if (b.useSPNEGO) { + // Secure the web server with kerberos + setupSpnegoFilter(b); + } + + initializeWebServer(b, threadPool.getMaxThreads()); + } + + private void initializeWebServer(final Builder b, int queueSize) { + // Set handling for low resource conditions. + final LowResourceMonitor low = new LowResourceMonitor(webServer); + low.setLowResourcesIdleTimeout(10000); + webServer.addBean(low); + + Connector connector = createChannelConnector(queueSize, b); webServer.addConnector(connector); RewriteHandler rwHandler = new RewriteHandler(); http://git-wip-us.apache.org/repos/asf/hive/blob/187eb760/common/src/test/org/apache/hadoop/hive/common/TestFileUtils.java ---------------------------------------------------------------------- diff --git a/common/src/test/org/apache/hadoop/hive/common/TestFileUtils.java b/common/src/test/org/apache/hadoop/hive/common/TestFileUtils.java index 03fcaeb..adc9b0c 100644 --- a/common/src/test/org/apache/hadoop/hive/common/TestFileUtils.java +++ b/common/src/test/org/apache/hadoop/hive/common/TestFileUtils.java @@ -213,7 +213,6 @@ public class TestFileUtils { Path copySrc = new Path("copySrc"); Path copyDst = new Path("copyDst"); HiveConf conf = new HiveConf(TestFileUtils.class); - conf.set(HiveConf.ConfVars.HIVE_WAREHOUSE_SUBDIR_INHERIT_PERMS.varname, "false"); FileSystem mockFs = mock(FileSystem.class); when(mockFs.getUri()).thenReturn(URI.create("hdfs:///")); http://git-wip-us.apache.org/repos/asf/hive/blob/187eb760/common/src/test/org/apache/hadoop/hive/common/TestValidReadTxnList.java ---------------------------------------------------------------------- diff --git a/common/src/test/org/apache/hadoop/hive/common/TestValidReadTxnList.java b/common/src/test/org/apache/hadoop/hive/common/TestValidReadTxnList.java index 6661158..00ee820 100644 --- a/common/src/test/org/apache/hadoop/hive/common/TestValidReadTxnList.java +++ b/common/src/test/org/apache/hadoop/hive/common/TestValidReadTxnList.java @@ -26,6 +26,7 @@ import java.io.DataOutputStream; import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; +import java.util.BitSet; /** * Tests for {@link ValidReadTxnList} @@ -34,9 +35,9 @@ public class TestValidReadTxnList { @Test public void noExceptions() throws Exception { - ValidTxnList txnList = new ValidReadTxnList(new long[0], 1, Long.MAX_VALUE); + ValidTxnList txnList = new ValidReadTxnList(new long[0], new BitSet(), 1, Long.MAX_VALUE); String str = txnList.writeToString(); - Assert.assertEquals("1:" + Long.MAX_VALUE + ":", str); + Assert.assertEquals("1:" + Long.MAX_VALUE + "::", str); ValidTxnList newList = new ValidReadTxnList(); newList.readFromString(str); Assert.assertTrue(newList.isTxnValid(1)); @@ -45,9 +46,9 @@ public class TestValidReadTxnList { @Test public void exceptions() throws Exception { - ValidTxnList txnList = new ValidReadTxnList(new long[]{2L,4L}, 5, 4L); + ValidTxnList txnList = new ValidReadTxnList(new long[]{2L,4L}, new BitSet(), 5, 4L); String str = txnList.writeToString(); - Assert.assertEquals("5:4:2:4", str); + Assert.assertEquals("5:4:2,4:", str); ValidTxnList newList = new ValidReadTxnList(); newList.readFromString(str); Assert.assertTrue(newList.isTxnValid(1)); @@ -62,7 +63,7 @@ public class TestValidReadTxnList { public void longEnoughToCompress() throws Exception { long[] exceptions = new long[1000]; for (int i = 0; i < 1000; i++) exceptions[i] = i + 100; - ValidTxnList txnList = new ValidReadTxnList(exceptions, 2000, 900); + ValidTxnList txnList = new ValidReadTxnList(exceptions, new BitSet(), 2000, 900); String str = txnList.writeToString(); ValidTxnList newList = new ValidReadTxnList(); newList.readFromString(str); @@ -76,7 +77,7 @@ public class TestValidReadTxnList { public void readWriteConfig() throws Exception { long[] exceptions = new long[1000]; for (int i = 0; i < 1000; i++) exceptions[i] = i + 100; - ValidTxnList txnList = new ValidReadTxnList(exceptions, 2000, 900); + ValidTxnList txnList = new ValidReadTxnList(exceptions, new BitSet(), 2000, 900); String str = txnList.writeToString(); Configuration conf = new Configuration(); conf.set(ValidTxnList.VALID_TXNS_KEY, str); @@ -89,4 +90,20 @@ public class TestValidReadTxnList { newConf.readFields(in); Assert.assertEquals(str, newConf.get(ValidTxnList.VALID_TXNS_KEY)); } + + @Test + public void testAbortedTxn() throws Exception { + long[] exceptions = {2L, 4L, 6L, 8L, 10L}; + BitSet bitSet = new BitSet(exceptions.length); + bitSet.set(0); // mark txn "2L" aborted + bitSet.set(3); // mark txn "8L" aborted + ValidTxnList txnList = new ValidReadTxnList(exceptions, bitSet, 11, 4L); + String str = txnList.writeToString(); + Assert.assertEquals("11:4:4,6,10:2,8", str); + Assert.assertTrue(txnList.isTxnAborted(2L)); + Assert.assertFalse(txnList.isTxnAborted(4L)); + Assert.assertFalse(txnList.isTxnAborted(6L)); + Assert.assertTrue(txnList.isTxnAborted(8L)); + Assert.assertFalse(txnList.isTxnAborted(10L)); + } } http://git-wip-us.apache.org/repos/asf/hive/blob/187eb760/common/src/test/org/apache/hadoop/hive/common/metrics/metrics2/TestCodahaleMetrics.java ---------------------------------------------------------------------- diff --git a/common/src/test/org/apache/hadoop/hive/common/metrics/metrics2/TestCodahaleMetrics.java b/common/src/test/org/apache/hadoop/hive/common/metrics/metrics2/TestCodahaleMetrics.java index aa4e75f..67f81d6 100644 --- a/common/src/test/org/apache/hadoop/hive/common/metrics/metrics2/TestCodahaleMetrics.java +++ b/common/src/test/org/apache/hadoop/hive/common/metrics/metrics2/TestCodahaleMetrics.java @@ -33,8 +33,6 @@ import org.junit.Before; import org.junit.Test; import java.io.File; -import java.nio.file.Files; -import java.nio.file.Paths; import java.util.concurrent.Callable; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -57,9 +55,12 @@ public class TestCodahaleMetrics { jsonReportFile = new File(workDir, "json_reporting"); jsonReportFile.delete(); + conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, "local"); conf.setVar(HiveConf.ConfVars.HIVE_METRICS_CLASS, CodahaleMetrics.class.getCanonicalName()); - conf.setVar(HiveConf.ConfVars.HIVE_METRICS_REPORTER, MetricsReporting.JSON_FILE.name() + "," + MetricsReporting.JMX.name()); + conf.setVar(HiveConf.ConfVars.HIVE_CODAHALE_METRICS_REPORTER_CLASSES, + "org.apache.hadoop.hive.common.metrics.metrics2.JsonFileMetricsReporter, " + + "org.apache.hadoop.hive.common.metrics.metrics2.JmxMetricsReporter"); conf.setVar(HiveConf.ConfVars.HIVE_METRICS_JSON_FILE_LOCATION, jsonReportFile.toString()); conf.setVar(HiveConf.ConfVars.HIVE_METRICS_JSON_FILE_INTERVAL, "100ms"); http://git-wip-us.apache.org/repos/asf/hive/blob/187eb760/common/src/test/org/apache/hadoop/hive/common/metrics/metrics2/TestCodahaleReportersConf.java ---------------------------------------------------------------------- diff --git a/common/src/test/org/apache/hadoop/hive/common/metrics/metrics2/TestCodahaleReportersConf.java b/common/src/test/org/apache/hadoop/hive/common/metrics/metrics2/TestCodahaleReportersConf.java new file mode 100644 index 0000000..d694e26 --- /dev/null +++ b/common/src/test/org/apache/hadoop/hive/common/metrics/metrics2/TestCodahaleReportersConf.java @@ -0,0 +1,145 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.common.metrics.metrics2; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import java.lang.reflect.InvocationTargetException; +import org.apache.hadoop.fs.CommonConfigurationKeysPublic; +import org.apache.hadoop.hive.common.metrics.MetricsTestUtils; +import org.apache.hadoop.hive.common.metrics.common.MetricsFactory; +import org.apache.hadoop.hive.conf.HiveConf; +import org.junit.After; +import org.junit.Assert; +import org.junit.Test; + +import java.io.File; + +/** + * Unit tests for Codahale reporter config backward compatibility + */ +public class TestCodahaleReportersConf { + + private static File workDir = new File(System.getProperty("test.tmp.dir")); + private static File jsonReportFile; + + @After + public void after() throws Exception { + MetricsFactory.close(); + } + + /** + * Tests that the deprecated HIVE_METRICS_REPORTER config is used if the HIVE_CODAHALE_METRICS_REPORTER_CLASSES is missing. + */ + @Test + public void testFallbackToDeprecatedConfig() throws Exception { + + HiveConf conf = new HiveConf(); + + jsonReportFile = new File(workDir, "json_reporting"); + jsonReportFile.delete(); + + conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, "local"); + conf.setVar(HiveConf.ConfVars.HIVE_METRICS_CLASS, CodahaleMetrics.class.getCanonicalName()); + conf.setVar(HiveConf.ConfVars.HIVE_METRICS_REPORTER, "JMX, JSON"); + conf.setVar(HiveConf.ConfVars.HIVE_METRICS_JSON_FILE_LOCATION, jsonReportFile.toString()); + conf.setVar(HiveConf.ConfVars.HIVE_METRICS_JSON_FILE_INTERVAL, "100ms"); + + MetricsFactory.init(conf); + + int runs = 5; + for (int i = 0; i < runs; i++) { + MetricsFactory.getInstance().incrementCounter("count2"); + } + + // we expect json file to be updated + byte[] jsonData = MetricsTestUtils.getFileData(jsonReportFile.getAbsolutePath(), 2000, 3); + ObjectMapper objectMapper = new ObjectMapper(); + + JsonNode rootNode = objectMapper.readTree(jsonData); + JsonNode countersNode = rootNode.path("counters"); + JsonNode methodCounterNode = countersNode.path("count2"); + JsonNode countNode = methodCounterNode.path("count"); + Assert.assertEquals(countNode.asInt(), 5); + } + + /** + * Tests that the deprecated HIVE_METRICS_REPORTER config is not used if + * HIVE_CODAHALE_METRICS_REPORTER_CLASSES is present. + * + * The deprecated config specifies json reporters whereas the newer one doesn't. Validates that + * the JSON file is not created. + */ + @Test + public void testNoFallback() throws Exception { + + HiveConf conf = new HiveConf(); + + jsonReportFile = new File(workDir, "json_reporting"); + jsonReportFile.delete(); + + conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, "local"); + conf.setVar(HiveConf.ConfVars.HIVE_METRICS_CLASS, CodahaleMetrics.class.getCanonicalName()); + conf.setVar(HiveConf.ConfVars.HIVE_METRICS_REPORTER, "JMX, JSON"); + conf.setVar(HiveConf.ConfVars.HIVE_CODAHALE_METRICS_REPORTER_CLASSES, + "org.apache.hadoop.hive.common.metrics.metrics2.JmxMetricsReporter"); + conf.setVar(HiveConf.ConfVars.HIVE_METRICS_JSON_FILE_LOCATION, jsonReportFile.toString()); + conf.setVar(HiveConf.ConfVars.HIVE_METRICS_JSON_FILE_INTERVAL, "100ms"); + + MetricsFactory.init(conf); + + int runs = 5; + for (int i = 0; i < runs; i++) { + MetricsFactory.getInstance().incrementCounter("count2"); + } + + Assert.assertFalse(jsonReportFile.exists()); + } + + /** + * Tests that the deprecated HIVE_METRICS_REPORTER config is not used if + * HIVE_CODAHALE_METRICS_REPORTER_CLASSES is present but incorrect. + * + * The deprecated config specifies json reporters whereas the newer one doesn't. Validates that + * the JSON file is not created. + */ + @Test + public void testNoFallbackOnIncorrectConf() throws Exception { + + HiveConf conf = new HiveConf(); + + jsonReportFile = new File(workDir, "json_reporting"); + jsonReportFile.delete(); + + conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, "local"); + conf.setVar(HiveConf.ConfVars.HIVE_METRICS_CLASS, CodahaleMetrics.class.getCanonicalName()); + conf.setVar(HiveConf.ConfVars.HIVE_METRICS_REPORTER, "JMX, JSON"); + conf.setVar(HiveConf.ConfVars.HIVE_CODAHALE_METRICS_REPORTER_CLASSES, + "org.apache.hadoop.hive.common.metrics.NonExistentReporter"); + conf.setVar(HiveConf.ConfVars.HIVE_METRICS_JSON_FILE_LOCATION, jsonReportFile.toString()); + conf.setVar(HiveConf.ConfVars.HIVE_METRICS_JSON_FILE_INTERVAL, "100ms"); + + try { + MetricsFactory.init(conf); + } catch (InvocationTargetException expectedException) { + + } + + Assert.assertFalse(jsonReportFile.exists()); + } +} http://git-wip-us.apache.org/repos/asf/hive/blob/187eb760/contrib/pom.xml ---------------------------------------------------------------------- diff --git a/contrib/pom.xml b/contrib/pom.xml index 6cc4931..7423e31 100644 --- a/contrib/pom.xml +++ b/contrib/pom.xml @@ -19,7 +19,7 @@ <parent> <groupId>org.apache.hive</groupId> <artifactId>hive</artifactId> - <version>2.2.0-SNAPSHOT</version> + <version>3.0.0-SNAPSHOT</version> <relativePath>../pom.xml</relativePath> </parent> http://git-wip-us.apache.org/repos/asf/hive/blob/187eb760/contrib/src/test/results/clientnegative/case_with_row_sequence.q.out ---------------------------------------------------------------------- diff --git a/contrib/src/test/results/clientnegative/case_with_row_sequence.q.out b/contrib/src/test/results/clientnegative/case_with_row_sequence.q.out index 9ee319f..cf92da6 100644 --- a/contrib/src/test/results/clientnegative/case_with_row_sequence.q.out +++ b/contrib/src/test/results/clientnegative/case_with_row_sequence.q.out @@ -10,18 +10,4 @@ POSTHOOK: query: create temporary function row_sequence as 'org.apache.hadoop.hive.contrib.udf.UDFRowSequence' POSTHOOK: type: CREATEFUNCTION POSTHOOK: Output: row_sequence -PREHOOK: query: SELECT CASE WHEN 3 > 2 THEN 10 WHEN row_sequence() > 5 THEN 20 ELSE 30 END -FROM src LIMIT 1 -PREHOOK: type: QUERY -PREHOOK: Input: default@src -#### A masked pattern was here #### -Execution failed with exit status: 2 -Obtaining error information - -Task failed! -Task ID: - Stage-1 - -Logs: - -FAILED: Execution Error, return code 2 from org.apache.hadoop.hive.ql.exec.mr.MapRedTask +FAILED: SemanticException Stateful expressions cannot be used inside of CASE http://git-wip-us.apache.org/repos/asf/hive/blob/187eb760/data/files/e011_01.txt ---------------------------------------------------------------------- diff --git a/data/files/e011_01.txt b/data/files/e011_01.txt new file mode 100644 index 0000000..92df12a --- /dev/null +++ b/data/files/e011_01.txt @@ -0,0 +1,4 @@ +12 +34 +56 +78 http://git-wip-us.apache.org/repos/asf/hive/blob/187eb760/data/files/events.txt ---------------------------------------------------------------------- diff --git a/data/files/events.txt b/data/files/events.txt new file mode 100644 index 0000000..e0bd0be --- /dev/null +++ b/data/files/events.txt @@ -0,0 +1,200 @@ +1111,20121121,1111,1,type1,eventData,session,full_uid,20121121,39,hq_change +1111,20121121,1111,2,type1,eventData,session,full_uid,20121121,39,hq_change +1111,20121121,1111,3,type1,eventData,session,full_uid,20121121,39,hq_change +1111,20121121,1111,4,type1,eventData,session,full_uid,20121121,39,hq_change +1111,20121121,1111,5,type1,eventData,session,full_uid,20121121,39,hq_change +1111,20121121,1111,6,type1,eventData,session,full_uid,20121121,39,hq_change +1111,20121121,1111,7,type1,eventData,session,full_uid,20121121,39,hq_change +1111,20121121,1111,8,type1,eventData,session,full_uid,20121121,39,hq_change +1111,20121121,1111,9,type1,eventData,session,full_uid,20121121,39,hq_change +1111,20121121,1111,10,type1,eventData,session,full_uid,20121121,39,hq_change +1111,20121121,1111,11,type1,eventData,session,full_uid,20121121,39,hq_change +1111,20121121,1111,12,type1,eventData,session,full_uid,20121121,39,hq_change +1111,20121121,1111,13,type1,eventData,session,full_uid,20121121,39,hq_change +1111,20121121,1111,14,type1,eventData,session,full_uid,20121121,39,hq_change +1111,20121121,1111,15,type1,eventData,session,full_uid,20121121,39,hq_change +1111,20121121,1111,16,type1,eventData,session,full_uid,20121121,39,hq_change +1111,20121121,1111,17,type1,eventData,session,full_uid,20121121,39,hq_change +1111,20121121,1111,18,type1,eventData,session,full_uid,20121121,39,hq_change +1111,20121121,1111,19,type1,eventData,session,full_uid,20121121,39,hq_change +1111,20121121,1111,20,type1,eventData,session,full_uid,20121121,39,hq_change +1111,20121121,1111,21,type1,eventData,session,full_uid,20121121,39,hq_change +1111,20121121,1111,22,type1,eventData,session,full_uid,20121121,39,hq_change +1111,20121121,1111,23,type1,eventData,session,full_uid,20121121,39,hq_change +1111,20121121,1111,24,type1,eventData,session,full_uid,20121121,39,hq_change +1111,20121121,1111,25,type1,eventData,session,full_uid,20121121,39,hq_change +1111,20121121,1111,26,type1,eventData,session,full_uid,20121121,39,hq_change +1111,20121121,1111,27,type1,eventData,session,full_uid,20121121,39,hq_change +1111,20121121,1111,28,type1,eventData,session,full_uid,20121121,39,hq_change +1111,20121121,1111,29,type1,eventData,session,full_uid,20121121,39,hq_change +1111,20121121,1111,30,type1,eventData,session,full_uid,20121121,39,hq_change +1111,20121121,1111,31,type1,eventData,session,full_uid,20121121,39,hq_change +1111,20121121,1111,32,type1,eventData,session,full_uid,20121121,39,hq_change +1111,20121121,1111,33,type1,eventData,session,full_uid,20121121,39,hq_change +1111,20121121,1111,34,type1,eventData,session,full_uid,20121121,39,hq_change +1111,20121121,1111,35,type1,eventData,session,full_uid,20121121,39,hq_change +1111,20121121,1111,36,type1,eventData,session,full_uid,20121121,39,hq_change +1111,20121121,1111,37,type1,eventData,session,full_uid,20121121,39,hq_change +1111,20121121,1111,38,type1,eventData,session,full_uid,20121121,39,hq_change +1111,20121121,1111,39,type1,eventData,session,full_uid,20121121,39,hq_change +1111,20121121,1111,40,type1,eventData,session,full_uid,20121121,39,hq_change +1111,20121121,1111,41,type1,eventData,session,full_uid,20121121,39,hq_change +1111,20121121,1111,42,type1,eventData,session,full_uid,20121121,39,hq_change +1111,20121121,1111,43,type1,eventData,session,full_uid,20121121,39,hq_change +1111,20121121,1111,44,type1,eventData,session,full_uid,20121121,39,hq_change +1111,20121121,1111,45,type1,eventData,session,full_uid,20121121,39,hq_change +1111,20121121,1111,46,type1,eventData,session,full_uid,20121121,39,hq_change +1111,20121121,1111,47,type1,eventData,session,full_uid,20121121,39,hq_change +1111,20121121,1111,48,type1,eventData,session,full_uid,20121121,39,hq_change +1111,20121121,1111,49,type1,eventData,session,full_uid,20121121,39,hq_change +1111,20121121,1111,50,type1,eventData,session,full_uid,20121121,39,hq_change +1111,20121021,1111,1,type1,eventData,session,full_uid,20121021,39,hq_change +1111,20121021,1111,2,type1,eventData,session,full_uid,20121021,39,hq_change +1111,20121021,1111,3,type1,eventData,session,full_uid,20121021,39,hq_change +1111,20121021,1111,4,type1,eventData,session,full_uid,20121021,39,hq_change +1111,20121021,1111,5,type1,eventData,session,full_uid,20121021,39,hq_change +1111,20121021,1111,6,type1,eventData,session,full_uid,20121021,39,hq_change +1111,20121021,1111,7,type1,eventData,session,full_uid,20121021,39,hq_change +1111,20121021,1111,8,type1,eventData,session,full_uid,20121021,39,hq_change +1111,20121021,1111,9,type1,eventData,session,full_uid,20121021,39,hq_change +1111,20121021,1111,10,type1,eventData,session,full_uid,20121021,39,hq_change +1111,20121021,1111,11,type1,eventData,session,full_uid,20121021,39,hq_change +1111,20121021,1111,12,type1,eventData,session,full_uid,20121021,39,hq_change +1111,20121021,1111,13,type1,eventData,session,full_uid,20121021,39,hq_change +1111,20121021,1111,14,type1,eventData,session,full_uid,20121021,39,hq_change +1111,20121021,1111,15,type1,eventData,session,full_uid,20121021,39,hq_change +1111,20121021,1111,16,type1,eventData,session,full_uid,20121021,39,hq_change +1111,20121021,1111,17,type1,eventData,session,full_uid,20121021,39,hq_change +1111,20121021,1111,18,type1,eventData,session,full_uid,20121021,39,hq_change +1111,20121021,1111,19,type1,eventData,session,full_uid,20121021,39,hq_change +1111,20121021,1111,20,type1,eventData,session,full_uid,20121021,39,hq_change +1111,20121021,1111,21,type1,eventData,session,full_uid,20121021,39,hq_change +1111,20121021,1111,22,type1,eventData,session,full_uid,20121021,39,hq_change +1111,20121021,1111,23,type1,eventData,session,full_uid,20121021,39,hq_change +1111,20121021,1111,24,type1,eventData,session,full_uid,20121021,39,hq_change +1111,20121021,1111,25,type1,eventData,session,full_uid,20121021,39,hq_change +1111,20121021,1111,26,type1,eventData,session,full_uid,20121021,39,hq_change +1111,20121021,1111,27,type1,eventData,session,full_uid,20121021,39,hq_change +1111,20121021,1111,28,type1,eventData,session,full_uid,20121021,39,hq_change +1111,20121021,1111,29,type1,eventData,session,full_uid,20121021,39,hq_change +1111,20121021,1111,30,type1,eventData,session,full_uid,20121021,39,hq_change +1111,20121021,1111,31,type1,eventData,session,full_uid,20121021,39,hq_change +1111,20121021,1111,32,type1,eventData,session,full_uid,20121021,39,hq_change +1111,20121021,1111,33,type1,eventData,session,full_uid,20121021,39,hq_change +1111,20121021,1111,34,type1,eventData,session,full_uid,20121021,39,hq_change +1111,20121021,1111,35,type1,eventData,session,full_uid,20121021,39,hq_change +1111,20121021,1111,36,type1,eventData,session,full_uid,20121021,39,hq_change +1111,20121021,1111,37,type1,eventData,session,full_uid,20121021,39,hq_change +1111,20121021,1111,38,type1,eventData,session,full_uid,20121021,39,hq_change +1111,20121021,1111,39,type1,eventData,session,full_uid,20121021,39,hq_change +1111,20121021,1111,40,type1,eventData,session,full_uid,20121021,39,hq_change +1111,20121021,1111,41,type1,eventData,session,full_uid,20121021,39,hq_change +1111,20121021,1111,42,type1,eventData,session,full_uid,20121021,39,hq_change +1111,20121021,1111,43,type1,eventData,session,full_uid,20121021,39,hq_change +1111,20121021,1111,44,type1,eventData,session,full_uid,20121021,39,hq_change +1111,20121021,1111,45,type1,eventData,session,full_uid,20121021,39,hq_change +1111,20121021,1111,46,type1,eventData,session,full_uid,20121021,39,hq_change +1111,20121021,1111,47,type1,eventData,session,full_uid,20121021,39,hq_change +1111,20121021,1111,48,type1,eventData,session,full_uid,20121021,39,hq_change +1111,20121021,1111,49,type1,eventData,session,full_uid,20121021,39,hq_change +1111,20121021,1111,50,type1,eventData,session,full_uid,20121021,39,hq_change +1111,20121221,1111,1,type1,eventData,session,full_uid,20121121,39,hq_change +1111,20121221,1111,2,type1,eventData,session,full_uid,20121121,39,hq_change +1111,20121221,1111,3,type1,eventData,session,full_uid,20121121,39,hq_change +1111,20121221,1111,4,type1,eventData,session,full_uid,20121121,39,hq_change +1111,20121221,1111,5,type1,eventData,session,full_uid,20121121,39,hq_change +1111,20121221,1111,6,type1,eventData,session,full_uid,20121121,39,hq_change +1111,20121221,1111,7,type1,eventData,session,full_uid,20121121,39,hq_change +1111,20121221,1111,8,type1,eventData,session,full_uid,20121121,39,hq_change +1111,20121221,1111,9,type1,eventData,session,full_uid,20121121,39,hq_change +1111,20121221,1111,10,type1,eventData,session,full_uid,20121121,39,hq_change +1111,20121221,1111,11,type1,eventData,session,full_uid,20121121,39,hq_change +1111,20121221,1111,12,type1,eventData,session,full_uid,20121121,39,hq_change +1111,20121221,1111,13,type1,eventData,session,full_uid,20121121,39,hq_change +1111,20121221,1111,14,type1,eventData,session,full_uid,20121121,39,hq_change +1111,20121221,1111,15,type1,eventData,session,full_uid,20121121,39,hq_change +1111,20121221,1111,16,type1,eventData,session,full_uid,20121121,39,hq_change +1111,20121221,1111,17,type1,eventData,session,full_uid,20121121,39,hq_change +1111,20121221,1111,18,type1,eventData,session,full_uid,20121121,39,hq_change +1111,20121221,1111,19,type1,eventData,session,full_uid,20121121,39,hq_change +1111,20121221,1111,20,type1,eventData,session,full_uid,20121121,39,hq_change +1111,20121221,1111,21,type1,eventData,session,full_uid,20121121,39,hq_change +1111,20121221,1111,22,type1,eventData,session,full_uid,20121121,39,hq_change +1111,20121221,1111,23,type1,eventData,session,full_uid,20121121,39,hq_change +1111,20121221,1111,24,type1,eventData,session,full_uid,20121121,39,hq_change +1111,20121221,1111,25,type1,eventData,session,full_uid,20121121,39,hq_change +1111,20121221,1111,26,type1,eventData,session,full_uid,20121121,39,hq_change +1111,20121221,1111,27,type1,eventData,session,full_uid,20121121,39,hq_change +1111,20121221,1111,28,type1,eventData,session,full_uid,20121121,39,hq_change +1111,20121221,1111,29,type1,eventData,session,full_uid,20121121,39,hq_change +1111,20121221,1111,30,type1,eventData,session,full_uid,20121121,39,hq_change +1111,20121221,1111,31,type1,eventData,session,full_uid,20121121,39,hq_change +1111,20121221,1111,32,type1,eventData,session,full_uid,20121121,39,hq_change +1111,20121221,1111,33,type1,eventData,session,full_uid,20121121,39,hq_change +1111,20121221,1111,34,type1,eventData,session,full_uid,20121121,39,hq_change +1111,20121221,1111,35,type1,eventData,session,full_uid,20121121,39,hq_change +1111,20121221,1111,36,type1,eventData,session,full_uid,20121121,39,hq_change +1111,20121221,1111,37,type1,eventData,session,full_uid,20121121,39,hq_change +1111,20121221,1111,38,type1,eventData,session,full_uid,20121121,39,hq_change +1111,20121221,1111,39,type1,eventData,session,full_uid,20121121,39,hq_change +1111,20121221,1111,40,type1,eventData,session,full_uid,20121121,39,hq_change +1111,20121221,1111,41,type1,eventData,session,full_uid,20121121,39,hq_change +1111,20121221,1111,42,type1,eventData,session,full_uid,20121121,39,hq_change +1111,20121221,1111,43,type1,eventData,session,full_uid,20121121,39,hq_change +1111,20121221,1111,44,type1,eventData,session,full_uid,20121121,39,hq_change +1111,20121221,1111,45,type1,eventData,session,full_uid,20121121,39,hq_change +1111,20121221,1111,46,type1,eventData,session,full_uid,20121121,39,hq_change +1111,20121221,1111,47,type1,eventData,session,full_uid,20121121,39,hq_change +1111,20121221,1111,48,type1,eventData,session,full_uid,20121121,39,hq_change +1111,20121221,1111,49,type1,eventData,session,full_uid,20121121,39,hq_change +1111,20121221,1111,50,type1,eventData,session,full_uid,20121121,39,hq_change +1111,20120921,1111,1,type1,eventData,session,full_uid,20120921,39,hq_change +1111,20120921,1111,2,type1,eventData,session,full_uid,20120921,39,hq_change +1111,20120921,1111,3,type1,eventData,session,full_uid,20120921,39,hq_change +1111,20120921,1111,4,type1,eventData,session,full_uid,20120921,39,hq_change +1111,20120921,1111,5,type1,eventData,session,full_uid,20120921,39,hq_change +1111,20120921,1111,6,type1,eventData,session,full_uid,20120921,39,hq_change +1111,20120921,1111,7,type1,eventData,session,full_uid,20120921,39,hq_change +1111,20120921,1111,8,type1,eventData,session,full_uid,20120921,39,hq_change +1111,20120921,1111,9,type1,eventData,session,full_uid,20120921,39,hq_change +1111,20120921,1111,10,type1,eventData,session,full_uid,20120921,39,hq_change +1111,20120921,1111,11,type1,eventData,session,full_uid,20120921,39,hq_change +1111,20120921,1111,12,type1,eventData,session,full_uid,20120921,39,hq_change +1111,20120921,1111,13,type1,eventData,session,full_uid,20120921,39,hq_change +1111,20120921,1111,14,type1,eventData,session,full_uid,20120921,39,hq_change +1111,20120921,1111,15,type1,eventData,session,full_uid,20120921,39,hq_change +1111,20120921,1111,16,type1,eventData,session,full_uid,20120921,39,hq_change +1111,20120921,1111,17,type1,eventData,session,full_uid,20120921,39,hq_change +1111,20120921,1111,18,type1,eventData,session,full_uid,20120921,39,hq_change +1111,20120921,1111,19,type1,eventData,session,full_uid,20120921,39,hq_change +1111,20120921,1111,20,type1,eventData,session,full_uid,20120921,39,hq_change +1111,20120921,1111,21,type1,eventData,session,full_uid,20120921,39,hq_change +1111,20120921,1111,22,type1,eventData,session,full_uid,20120921,39,hq_change +1111,20120921,1111,23,type1,eventData,session,full_uid,20120921,39,hq_change +1111,20120921,1111,24,type1,eventData,session,full_uid,20120921,39,hq_change +1111,20120921,1111,25,type1,eventData,session,full_uid,20120921,39,hq_change +1111,20120921,1111,26,type1,eventData,session,full_uid,20120921,39,hq_change +1111,20120921,1111,27,type1,eventData,session,full_uid,20120921,39,hq_change +1111,20120921,1111,28,type1,eventData,session,full_uid,20120921,39,hq_change +1111,20120921,1111,29,type1,eventData,session,full_uid,20120921,39,hq_change +1111,20120921,1111,30,type1,eventData,session,full_uid,20120921,39,hq_change +1111,20120921,1111,31,type1,eventData,session,full_uid,20120921,39,hq_change +1111,20120921,1111,32,type1,eventData,session,full_uid,20120921,39,hq_change +1111,20120921,1111,33,type1,eventData,session,full_uid,20120921,39,hq_change +1111,20120921,1111,34,type1,eventData,session,full_uid,20120921,39,hq_change +1111,20120921,1111,35,type1,eventData,session,full_uid,20120921,39,hq_change +1111,20120921,1111,36,type1,eventData,session,full_uid,20120921,39,hq_change +1111,20120921,1111,37,type1,eventData,session,full_uid,20120921,39,hq_change +1111,20120921,1111,38,type1,eventData,session,full_uid,20120921,39,hq_change +1111,20120921,1111,39,type1,eventData,session,full_uid,20120921,39,hq_change +1111,20120921,1111,40,type1,eventData,session,full_uid,20120921,39,hq_change +1111,20120921,1111,41,type1,eventData,session,full_uid,20120921,39,hq_change +1111,20120921,1111,42,type1,eventData,session,full_uid,20120921,39,hq_change +1111,20120921,1111,43,type1,eventData,session,full_uid,20120921,39,hq_change +1111,20120921,1111,44,type1,eventData,session,full_uid,20120921,39,hq_change +1111,20120921,1111,45,type1,eventData,session,full_uid,20120921,39,hq_change +1111,20120921,1111,46,type1,eventData,session,full_uid,20120921,39,hq_change +1111,20120921,1111,47,type1,eventData,session,full_uid,20120921,39,hq_change +1111,20120921,1111,48,type1,eventData,session,full_uid,20120921,39,hq_change +1111,20120921,1111,49,type1,eventData,session,full_uid,20120921,39,hq_change +1111,20120921,1111,50,type1,eventData,session,full_uid,20120921,39,hq_change http://git-wip-us.apache.org/repos/asf/hive/blob/187eb760/data/files/tpcds-perf/metastore_export/csv/TABLE_PARAMS.txt ---------------------------------------------------------------------- diff --git a/data/files/tpcds-perf/metastore_export/csv/TABLE_PARAMS.txt b/data/files/tpcds-perf/metastore_export/csv/TABLE_PARAMS.txt deleted file mode 100644 index 78020f1..0000000 --- a/data/files/tpcds-perf/metastore_export/csv/TABLE_PARAMS.txt +++ /dev/null @@ -1,143 +0,0 @@ -_store_@COLUMN_STATS_ACCURATE@{"COLUMN_STATS":{"s_store_sk":"true","s_store_id":"true","s_rec_start_date":"true","s_rec_end_date":"true","s_closed_date_sk":"true","s_store_name":"true","s_number_employees":"true","s_floor_space":"true","s_hours":"true","s_manager":"true","s_market_id":"true","s_geography_class":"true","s_market_desc":"true","s_market_manager":"true","s_division_id":"true","s_division_name":"true","s_company_id":"true","s_company_name":"true","s_street_number":"true","s_street_name":"true","s_street_type":"true","s_suite_number":"true","s_city":"true","s_county":"true","s_state":"true","s_zip":"true","s_country":"true","s_gmt_offset":"true","s_tax_precentage":"true"},"BASIC_STATS":"true"} -_store_@numFiles@1 -_store_@numRows@1704 -_store_@rawDataSize@3256276 -_store_@totalSize@101707 -_store_@transient_lastDdlTime@1434562098 -_call_center_@COLUMN_STATS_ACCURATE@{"COLUMN_STATS":{"cc_call_center_sk":"true","cc_call_center_id":"true","cc_rec_start_date":"true","cc_rec_end_date":"true","cc_closed_date_sk":"true","cc_open_date_sk":"true","cc_name":"true","cc_class":"true","cc_employees":"true","cc_sq_ft":"true","cc_hours":"true","cc_manager":"true","cc_mkt_id":"true","cc_mkt_class":"true","cc_mkt_desc":"true","cc_market_manager":"true","cc_division":"true","cc_division_name":"true","cc_company":"true","cc_company_name":"true","cc_street_number":"true","cc_street_name":"true","cc_street_type":"true","cc_suite_number":"true","cc_city":"true","cc_county":"true","cc_state":"true","cc_zip":"true","cc_country":"true","cc_gmt_offset":"true","cc_tax_percentage":"true"},"BASIC_STATS":"true"} -_call_center_@numFiles@1 -_call_center_@numRows@60 -_call_center_@rawDataSize@122700 -_call_center_@totalSize@10347 -_call_center_@transient_lastDdlTime@1434561922 -_catalog_page_@COLUMN_STATS_ACCURATE@{"COLUMN_STATS":{"cp_catalog_page_sk":"true","cp_catalog_page_id":"true","cp_start_date_sk":"true","cp_end_date_sk":"true","cp_department":"true","cp_catalog_number":"true","cp_catalog_page_number":"true","cp_description":"true","cp_type":"true"},"BASIC_STATS":"true"} -_catalog_page_@numFiles@1 -_catalog_page_@numRows@46000 -_catalog_page_@rawDataSize@21198808 -_catalog_page_@totalSize@1576662 -_catalog_page_@transient_lastDdlTime@1434561925 -_customer_@COLUMN_STATS_ACCURATE@{"COLUMN_STATS":{"c_customer_sk":"true","c_customer_id":"true","c_current_cdemo_sk":"true","c_current_hdemo_sk":"true","c_current_addr_sk":"true","c_first_shipto_date_sk":"true","c_first_sales_date_sk":"true","c_salutation":"true","c_first_name":"true","c_last_name":"true","c_preferred_cust_flag":"true","c_birth_day":"true","c_birth_month":"true","c_birth_year":"true","c_birth_country":"true","c_login":"true","c_email_address":"true","c_last_review_date":"true"},"BASIC_STATS":"true"} -_customer_@numFiles@538 -_customer_@numRows@80000000 -_customer_@rawDataSize@68801615852 -_customer_@totalSize@3143935054 -_customer_@transient_lastDdlTime@1434561966 -_customer_address_@COLUMN_STATS_ACCURATE@{"COLUMN_STATS":{"ca_address_sk":"true","ca_address_id":"true","ca_street_number":"true","ca_street_name":"true","ca_street_type":"true","ca_suite_number":"true","ca_city":"true","ca_county":"true","ca_state":"true","ca_zip":"true","ca_country":"true","ca_gmt_offset":"true","ca_location_type":"true"},"BASIC_STATS":"true"} -_customer_address_@numFiles@274 -_customer_address_@numRows@40000000 -_customer_address_@rawDataSize@40595195284 -_customer_address_@totalSize@530195843 -_customer_address_@transient_lastDdlTime@1434561994 -_customer_demographics_@COLUMN_STATS_ACCURATE@{"COLUMN_STATS":{"cd_demo_sk":"true","cd_gender":"true","cd_marital_status":"true","cd_education_status":"true","cd_purchase_estimate":"true","cd_credit_rating":"true","cd_dep_count":"true","cd_dep_employed_count":"true","cd_dep_college_count":"true"},"BASIC_STATS":"true"} -_customer_demographics_@numFiles@8 -_customer_demographics_@numRows@1861800 -_customer_demographics_@rawDataSize@717186159 -_customer_demographics_@totalSize@323062 -_customer_demographics_@transient_lastDdlTime@1434562071 -_date_dim_@COLUMN_STATS_ACCURATE@{"COLUMN_STATS":{"d_date_sk":"true","d_date_id":"true","d_date":"true","d_month_seq":"true","d_week_seq":"true","d_quarter_seq":"true","d_year":"true","d_dow":"true","d_moy":"true","d_dom":"true","d_qoy":"true","d_fy_year":"true","d_fy_quarter_seq":"true","d_fy_week_seq":"true","d_day_name":"true","d_quarter_name":"true","d_holiday":"true","d_weekend":"true","d_following_holiday":"true","d_first_dom":"true","d_last_dom":"true","d_same_day_ly":"true","d_same_day_lq":"true","d_current_day":"true","d_current_week":"true","d_current_month":"true","d_current_quarter":"true","d_current_year":"true"},"BASIC_STATS":"true"} -_date_dim_@numFiles@1 -_date_dim_@numRows@73049 -_date_dim_@rawDataSize@81741831 -_date_dim_@totalSize@362925 -_date_dim_@transient_lastDdlTime@1434562075 -_household_demographics_@COLUMN_STATS_ACCURATE@{"COLUMN_STATS":{"hd_demo_sk":"true","hd_income_band_sk":"true","hd_buy_potential":"true","hd_dep_count":"true","hd_vehicle_count":"true"},"BASIC_STATS":"true"} -_household_demographics_@numFiles@1 -_household_demographics_@numRows@7200 -_household_demographics_@rawDataSize@770400 -_household_demographics_@totalSize@901 -_household_demographics_@transient_lastDdlTime@1434562078 -_income_band_@COLUMN_STATS_ACCURATE@{"COLUMN_STATS":{"ib_income_band_sk":"true","ib_lower_bound":"true","ib_upper_bound":"true"},"BASIC_STATS":"true"} -_income_band_@numFiles@1 -_income_band_@numRows@20 -_income_band_@rawDataSize@240 -_income_band_@totalSize@399 -_income_band_@transient_lastDdlTime@1434562081 -_item_@COLUMN_STATS_ACCURATE@{"COLUMN_STATS":{"i_item_sk":"true","i_item_id":"true","i_rec_start_date":"true","i_rec_end_date":"true","i_item_desc":"true","i_current_price":"true","i_wholesale_cost":"true","i_brand_id":"true","i_brand":"true","i_class_id":"true","i_class":"true","i_category_id":"true","i_category":"true","i_manufact_id":"true","i_manufact":"true","i_size":"true","i_formulation":"true","i_color":"true","i_units":"true","i_container":"true","i_manager_id":"true","i_product_name":"true"},"BASIC_STATS":"true"} -_item_@numFiles@9 -_item_@numRows@462000 -_item_@rawDataSize@663560457 -_item_@totalSize@29760748 -_item_@transient_lastDdlTime@1434562091 -_promotion_@COLUMN_STATS_ACCURATE@{"COLUMN_STATS":{"p_promo_sk":"true","p_promo_id":"true","p_start_date_sk":"true","p_end_date_sk":"true","p_item_sk":"true","p_cost":"true","p_response_target":"true","p_promo_name":"true","p_channel_dmail":"true","p_channel_email":"true","p_channel_catalog":"true","p_channel_tv":"true","p_channel_radio":"true","p_channel_press":"true","p_channel_event":"true","p_channel_demo":"true","p_channel_details":"true","p_purpose":"true","p_discount_active":"true"},"BASIC_STATS":"true"} -_promotion_@numFiles@1 -_promotion_@numRows@2300 -_promotion_@rawDataSize@2713420 -_promotion_@totalSize@63964 -_promotion_@transient_lastDdlTime@1434562093 -_reason_@COLUMN_STATS_ACCURATE@{"COLUMN_STATS":{"r_reason_sk":"true","r_reason_id":"true","r_reason_desc":"true"},"BASIC_STATS":"true"} -_reason_@numFiles@1 -_reason_@numRows@72 -_reason_@rawDataSize@14400 -_reason_@totalSize@1024 -_reason_@transient_lastDdlTime@1434562095 -_ship_mode_@COLUMN_STATS_ACCURATE@{"COLUMN_STATS":{"sm_ship_mode_sk":"true","sm_ship_mode_id":"true","sm_type":"true","sm_code":"true","sm_carrier":"true","sm_contract":"true"},"BASIC_STATS":"true"} -_ship_mode_@numFiles@0 -_ship_mode_@numRows@0 -_ship_mode_@rawDataSize@0 -_ship_mode_@totalSize@0 -_ship_mode_@transient_lastDdlTime@1434562097 -_time_dim_@COLUMN_STATS_ACCURATE@{"COLUMN_STATS":{"t_time_sk":"true","t_time_id":"true","t_time":"true","t_hour":"true","t_minute":"true","t_second":"true","t_am_pm":"true","t_shift":"true","t_sub_shift":"true","t_meal_time":"true"},"BASIC_STATS":"true"} -_time_dim_@numFiles@1 -_time_dim_@numRows@86400 -_time_dim_@rawDataSize@40694400 -_time_dim_@totalSize@133902 -_time_dim_@transient_lastDdlTime@1434562099 -_warehouse_@COLUMN_STATS_ACCURATE@{"COLUMN_STATS":{"w_warehouse_sk":"true","w_warehouse_id":"true","w_warehouse_name":"true","w_warehouse_sq_ft":"true","w_street_number":"true","w_street_name":"true","w_street_type":"true","w_suite_number":"true","w_city":"true","w_county":"true","w_state":"true","w_zip":"true","w_country":"true","w_gmt_offset":"true"},"BASIC_STATS":"true"} -_warehouse_@numFiles@1 -_warehouse_@numRows@27 -_warehouse_@rawDataSize@27802 -_warehouse_@totalSize@2971 -_warehouse_@transient_lastDdlTime@1434562102 -_web_page_@COLUMN_STATS_ACCURATE@{"COLUMN_STATS":{"wp_web_page_sk":"true","wp_web_page_id":"true","wp_rec_start_date":"true","wp_rec_end_date":"true","wp_creation_date_sk":"true","wp_access_date_sk":"true","wp_autogen_flag":"true","wp_customer_sk":"true","wp_url":"true","wp_type":"true","wp_char_count":"true","wp_link_count":"true","wp_image_count":"true","wp_max_ad_count":"true"},"BASIC_STATS":"true"} -_web_page_@numFiles@1 -_web_page_@numRows@4602 -_web_page_@rawDataSize@2696178 -_web_page_@totalSize@50572 -_web_page_@transient_lastDdlTime@1434562104 -_web_site_@COLUMN_STATS_ACCURATE@{"COLUMN_STATS":{"web_site_sk":"true","web_site_id":"true","web_rec_start_date":"true","web_rec_end_date":"true","web_name":"true","web_open_date_sk":"true","web_close_date_sk":"true","web_class":"true","web_manager":"true","web_mkt_id":"true","web_mkt_class":"true","web_mkt_desc":"true","web_market_manager":"true","web_company_id":"true","web_company_name":"true","web_street_number":"true","web_street_name":"true","web_street_type":"true","web_suite_number":"true","web_city":"true","web_county":"true","web_state":"true","web_zip":"true","web_country":"true","web_gmt_offset":"true","web_tax_percentage":"true"},"BASIC_STATS":"true"} -_web_site_@numFiles@1 -_web_site_@numRows@84 -_web_site_@rawDataSize@155408 -_web_site_@totalSize@11271 -_web_site_@transient_lastDdlTime@1434562107 -_catalog_returns_@COLUMN_STATS_ACCURATE@{"COLUMN_STATS":{"cr_returned_date_sk":"true","cr_returned_time_sk":"true","cr_item_sk":"true","cr_refunded_customer_sk":"true","cr_refunded_cdemo_sk":"true","cr_refunded_hdemo_sk":"true","cr_refunded_addr_sk":"true","cr_returning_customer_sk":"true","cr_returning_cdemo_sk":"true","cr_returning_hdemo_sk":"true","cr_returning_addr_sk":"true","cr_call_center_sk":"true","cr_catalog_page_sk":"true","cr_ship_mode_sk":"true","cr_warehouse_sk":"true","cr_reason_sk":"true","cr_order_number":"true","cr_return_quantity":"true","cr_return_amount":"true","cr_return_tax":"true","cr_return_amt_inc_tax":"true","cr_fee":"true","cr_return_ship_cost":"true","cr_refunded_cash":"true","cr_reversed_charge":"true","cr_store_credit":"true","cr_net_loss":"true"},"BASIC_STATS":"true"} -_catalog_returns_@numFiles@400 -_catalog_returns_@numRows@28798881 -_catalog_returns_@rawDataSize@3057234680 -_catalog_returns_@totalSize@1651022096 -_catalog_sales_@COLUMN_STATS_ACCURATE@{"COLUMN_STATS":{"cs_sold_date_sk":"true","cs_sold_time_sk":"true","cs_ship_date_sk":"true","cs_bill_customer_sk":"true","cs_bill_cdemo_sk":"true","cs_bill_hdemo_sk":"true","cs_bill_addr_sk":"true","cs_ship_customer_sk":"true","cs_ship_cdemo_sk":"true","cs_ship_hdemo_sk":"true","cs_ship_addr_sk":"true","cs_call_center_sk":"true","cs_catalog_page_sk":"true","cs_ship_mode_sk":"true","cs_warehouse_sk":"true","cs_item_sk":"true","cs_promo_sk":"true","cs_order_number":"true","cs_quantity":"true","cs_wholesale_cost":"true","cs_list_price":"true","cs_sales_price":"true","cs_ext_discount_amt":"true","cs_ext_sales_price":"true","cs_ext_wholesale_cost":"true","cs_ext_list_price":"true","cs_ext_tax":"true","cs_coupon_amt":"true","cs_ext_ship_cost":"true","cs_net_paid":"true","cs_net_paid_inc_tax":"true","cs_net_paid_inc_ship":"true","cs_net_paid_inc_ship_tax":"true","cs_net_profit":"true"},"BASIC_STATS":"true"} -_catalog_sales_@numFiles@552 -_catalog_sales_@numRows@287989836 -_catalog_sales_@rawDataSize@38999608952 -_catalog_sales_@totalSize@16430853294 -_catalog_sales_@transient_lastDdlTime@1434700893 -_inventory_@COLUMN_STATS_ACCURATE@{"COLUMN_STATS":{"inv_date_sk":"true","inv_item_sk":"true","inv_warehouse_sk":"true","inv_quantity_on_hand":"true"},"BASIC_STATS":"true"} -_inventory_@numFiles@51 -_inventory_@numRows@37584000 -_inventory_@rawDataSize@593821104 -_inventory_@totalSize@58323594 -_inventory_@transient_lastDdlTime@1434681166 -_store_sales_@COLUMN_STATS_ACCURATE@{"COLUMN_STATS":{"ss_sold_date_sk":"true","ss_sold_time_sk":"true","ss_item_sk":"true","ss_customer_sk":"true","ss_cdemo_sk":"true","ss_hdemo_sk":"true","ss_addr_sk":"true","ss_store_sk":"true","ss_promo_sk":"true","ss_ticket_number":"true","ss_quantity":"true","ss_wholesale_cost":"true","ss_list_price":"true","ss_sales_price":"true","ss_ext_discount_amt":"true","ss_ext_sales_price":"true","ss_ext_wholesale_cost":"true","ss_ext_list_price":"true","ss_ext_tax":"true","ss_coupon_amt":"true","ss_net_paid":"true","ss_net_paid_inc_tax":"true","ss_net_profit":"true"},"BASIC_STATS":"true"} -_store_sales_@numFiles@600 -_store_sales_@numRows@575995635 -_store_sales_@rawDataSize@50814502088 -_store_sales_@totalSize@22288384284 -_store_sales_@transient_lastDdlTime@1434700760 -_web_sales_@COLUMN_STATS_ACCURATE@{"COLUMN_STATS":{"ws_sold_date_sk":"true","ws_sold_time_sk":"true","ws_ship_date_sk":"true","ws_item_sk":"true","ws_bill_customer_sk":"true","ws_bill_cdemo_sk":"true","ws_bill_hdemo_sk":"true","ws_bill_addr_sk":"true","ws_ship_customer_sk":"true","ws_ship_cdemo_sk":"true","ws_ship_hdemo_sk":"true","ws_ship_addr_sk":"true","ws_web_page_sk":"true","ws_web_site_sk":"true","ws_ship_mode_sk":"true","ws_warehouse_sk":"true","ws_promo_sk":"true","ws_order_number":"true","ws_quantity":"true","ws_wholesale_cost":"true","ws_list_price":"true","ws_sales_price":"true","ws_ext_discount_amt":"true","ws_ext_sales_price":"true","ws_ext_wholesale_cost":"true","ws_ext_list_price":"true","ws_ext_tax":"true","ws_coupon_amt":"true","ws_ext_ship_cost":"true","ws_net_paid":"true","ws_net_paid_inc_tax":"true","ws_net_paid_inc_ship":"true","ws_net_paid_inc_ship_tax":"true","ws_net_profit":"true"},"BASIC_STATS":"true"} -_web_sales_@numFiles@529 -_web_sales_@numRows@144002668 -_web_sales_@rawDataSize@19580198212 -_web_sales_@totalSize@8304889016 -_web_sales_@transient_lastDdlTime@1434700829 -_web_returns_@COLUMN_STATS_ACCURATE@{"COLUMN_STATS":{"wr_returned_date_sk":"true","wr_returned_time_sk":"true","wr_item_sk":"true","wr_refunded_customer_sk":"true","wr_refunded_cdemo_sk":"true","wr_refunded_hdemo_sk":"true","wr_refunded_addr_sk":"true","wr_returning_customer_sk":"true","wr_returning_cdemo_sk":"true","wr_returning_hdemo_sk":"true","wr_returning_addr_sk":"true","wr_web_page_sk":"true","wr_reason_sk":"true","wr_order_number":"true","wr_return_quantity":"true","wr_return_amt":"true","wr_return_tax":"true","wr_return_amt_inc_tax":"true","wr_fee":"true","wr_return_ship_cost":"true","wr_refunded_cash":"true","wr_reversed_charge":"true","wr_account_credit":"true","wr_net_loss":"true"},"BASIC_STATS":"true"} -_web_returns_@numFiles@135 -_web_returns_@numRows@14398467 -_web_returns_@rawDataSize@1325194184 -_web_returns_@totalSize@827734274 -_web_returns_@transient_lastDdlTime@1434700842 -_store_returns_@COLUMN_STATS_ACCURATE@{"COLUMN_STATS":{"sr_returned_date_sk":"true","sr_return_time_sk":"true","sr_item_sk":"true","sr_customer_sk":"true","sr_cdemo_sk":"true","sr_hdemo_sk":"true","sr_addr_sk":"true","sr_store_sk":"true","sr_reason_sk":"true","sr_ticket_number":"true","sr_return_quantity":"true","sr_return_amt":"true","sr_return_tax":"true","sr_return_amt_inc_tax":"true","sr_fee":"true","sr_return_ship_cost":"true","sr_refunded_cash":"true","sr_reversed_charge":"true","sr_store_credit":"true","sr_net_loss":"true"},"BASIC_STATS":"true"} -_store_returns_@numFiles@573 -_store_returns_@numRows@57591150 -_store_returns_@rawDataSize@4462194832 -_store_returns_@totalSize@2683203627 -_store_returns_@transient_lastDdlTime@1434700790
