Author: xuefu
Date: Mon Feb 2 21:10:08 2015
New Revision: 1656573
URL: http://svn.apache.org/r1656573
Log:
HIVE-9211: Research on build mini HoS cluster on YARN for unit test[Spark
Branch] (Chengxiang via Xuefu)
Added:
hive/branches/spark/data/conf/spark/standalone/
hive/branches/spark/data/conf/spark/standalone/hive-site.xml
hive/branches/spark/data/conf/spark/yarn-client/
hive/branches/spark/data/conf/spark/yarn-client/hive-site.xml
hive/branches/spark/ql/src/test/results/clientpositive/spark/bucket5.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/bucket6.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/bucketizedhiveinputformat.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/constprog_partitioner.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/empty_dir_in_table.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/external_table_with_space_in_location_path.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/file_with_header_footer.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/import_exported_table.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/index_bitmap3.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/index_bitmap_auto.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/infer_bucket_sort_bucketed_table.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/infer_bucket_sort_map_operators.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/infer_bucket_sort_merge.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/infer_bucket_sort_num_buckets.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/infer_bucket_sort_reducers_power_two.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/input16_cc.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/list_bucket_dml_10.q.java1.7.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/load_fs2.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/load_hdfs_file_with_space_in_the_name.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/parallel_orderby.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/ql_rewrite_gbtoidx.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/ql_rewrite_gbtoidx_cbo_1.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/quotedid_smb.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/reduce_deduplicate.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/remote_script.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/root_dir_external_table.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/schemeAuthority.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/schemeAuthority2.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/temp_table_external.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/truncate_column_buckets.q.out
hive/branches/spark/ql/src/test/results/clientpositive/spark/uber_reduce.q.out
hive/branches/spark/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/MiniSparkOnYARNCluster.java
Removed:
hive/branches/spark/data/conf/spark/hive-site.xml
Modified:
hive/branches/spark/itests/pom.xml
hive/branches/spark/itests/qtest-spark/pom.xml
hive/branches/spark/itests/src/test/resources/testconfiguration.properties
hive/branches/spark/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/RemoteHiveSparkClient.java
hive/branches/spark/shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java
hive/branches/spark/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
hive/branches/spark/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java
hive/branches/spark/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientImpl.java
Added: hive/branches/spark/data/conf/spark/standalone/hive-site.xml
URL:
http://svn.apache.org/viewvc/hive/branches/spark/data/conf/spark/standalone/hive-site.xml?rev=1656573&view=auto
==============================================================================
--- hive/branches/spark/data/conf/spark/standalone/hive-site.xml (added)
+++ hive/branches/spark/data/conf/spark/standalone/hive-site.xml Mon Feb 2
21:10:08 2015
@@ -0,0 +1,233 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<configuration>
+
+<!-- Hive Configuration can either be stored in this file or in the hadoop
configuration files -->
+<!-- that are implied by Hadoop setup variables.
-->
+<!-- Aside from Hadoop setup variables - this file is provided as a
convenience so that Hive -->
+<!-- users do not have to edit hadoop configuration files (that may be managed
as a centralized -->
+<!-- resource).
-->
+
+<!-- Hive Execution Parameters -->
+<property>
+ <name>hadoop.tmp.dir</name>
+ <value>${test.tmp.dir}/hadoop-tmp</value>
+ <description>A base for other temporary directories.</description>
+</property>
+
+<property>
+ <name>hive.exec.scratchdir</name>
+ <value>${test.tmp.dir}/scratchdir</value>
+ <description>Scratch space for Hive jobs</description>
+</property>
+
+<property>
+ <name>hive.exec.local.scratchdir</name>
+ <value>${test.tmp.dir}/localscratchdir/</value>
+ <description>Local scratch space for Hive jobs</description>
+</property>
+
+<property>
+ <name>javax.jdo.option.ConnectionURL</name>
+
<value>jdbc:derby:;databaseName=${test.tmp.dir}/junit_metastore_db;create=true</value>
+</property>
+
+<property>
+ <name>hive.stats.dbconnectionstring</name>
+
<value>jdbc:derby:;databaseName=${test.tmp.dir}/TempStatsStore;create=true</value>
+</property>
+
+
+<property>
+ <name>javax.jdo.option.ConnectionDriverName</name>
+ <value>org.apache.derby.jdbc.EmbeddedDriver</value>
+</property>
+
+<property>
+ <name>javax.jdo.option.ConnectionUserName</name>
+ <value>APP</value>
+</property>
+
+<property>
+ <name>javax.jdo.option.ConnectionPassword</name>
+ <value>mine</value>
+</property>
+
+<property>
+ <!-- this should eventually be deprecated since the metastore should supply
this -->
+ <name>hive.metastore.warehouse.dir</name>
+ <value>${test.warehouse.dir}</value>
+ <description></description>
+</property>
+
+<property>
+ <name>hive.metastore.metadb.dir</name>
+ <value>file://${test.tmp.dir}/metadb/</value>
+ <description>
+ Required by metastore server or if the uris argument below is not supplied
+ </description>
+</property>
+
+<property>
+ <name>test.log.dir</name>
+ <value>${test.tmp.dir}/log/</value>
+ <description></description>
+</property>
+
+<property>
+ <name>test.data.files</name>
+ <value>${hive.root}/data/files</value>
+ <description></description>
+</property>
+
+<property>
+ <name>hive.jar.path</name>
+
<value>${maven.local.repository}/org/apache/hive/hive-exec/${hive.version}/hive-exec-${hive.version}.jar</value>
+ <description></description>
+</property>
+
+<property>
+ <name>test.data.scripts</name>
+ <value>${hive.root}/data/scripts</value>
+ <description></description>
+</property>
+
+<property>
+ <name>hive.metastore.rawstore.impl</name>
+ <value>org.apache.hadoop.hive.metastore.ObjectStore</value>
+ <description>Name of the class that implements
org.apache.hadoop.hive.metastore.rawstore interface. This class is used to
store and retrieval of raw metadata objects such as table,
database</description>
+</property>
+
+<property>
+ <name>hive.querylog.location</name>
+ <value>${test.tmp.dir}/tmp</value>
+ <description>Location of the structured hive logs</description>
+</property>
+
+<property>
+ <name>hive.exec.pre.hooks</name>
+ <value>org.apache.hadoop.hive.ql.hooks.PreExecutePrinter,
org.apache.hadoop.hive.ql.hooks.EnforceReadOnlyTables</value>
+ <description>Pre Execute Hook for Tests</description>
+</property>
+
+<property>
+ <name>hive.exec.post.hooks</name>
+ <value>org.apache.hadoop.hive.ql.hooks.PostExecutePrinter</value>
+ <description>Post Execute Hook for Tests</description>
+</property>
+
+<property>
+ <name>hive.support.concurrency</name>
+ <value>false</value>
+ <description>Whether hive supports concurrency or not. A zookeeper instance
must be up and running for the default hive lock manager to support read-write
locks.</description>
+</property>
+
+<property>
+ <name>fs.pfile.impl</name>
+ <value>org.apache.hadoop.fs.ProxyLocalFileSystem</value>
+ <description>A proxy for local file system used for cross file system
testing</description>
+</property>
+
+<property>
+ <name>hive.exec.mode.local.auto</name>
+ <value>false</value>
+ <description>
+ Let hive determine whether to run in local mode automatically
+ Disabling this for tests so that minimr is not affected
+ </description>
+</property>
+
+<property>
+ <name>hive.auto.convert.join</name>
+ <value>false</value>
+ <description>Whether Hive enable the optimization about converting common
join into mapjoin based on the input file size</description>
+</property>
+
+<property>
+ <name>hive.ignore.mapjoin.hint</name>
+ <value>false</value>
+ <description>Whether Hive ignores the mapjoin hint</description>
+</property>
+
+<property>
+ <name>io.sort.mb</name>
+ <value>10</value>
+</property>
+
+<property>
+ <name>hive.input.format</name>
+ <value>org.apache.hadoop.hive.ql.io.CombineHiveInputFormat</value>
+ <description>The default input format, if it is not specified, the system
assigns it. It is set to HiveInputFormat for hadoop versions 17, 18 and 19,
whereas it is set to CombineHiveInputFormat for hadoop 20. The user can always
overwrite it - if there is a bug in CombineHiveInputFormat, it can always be
manually set to HiveInputFormat. </description>
+</property>
+
+<property>
+ <name>hive.default.rcfile.serde</name>
+ <value>org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe</value>
+ <description>The default SerDe hive will use for the rcfile
format</description>
+</property>
+
+<property>
+ <name>hive.stats.dbclass</name>
+ <value>fs</value>
+ <description>The default storatge that stores temporary hive statistics.
Currently, jdbc, hbase and counter type is supported</description>
+</property>
+
+<property>
+ <name>hive.execution.engine</name>
+ <value>spark</value>
+ <description>Chooses execution engine. Options are: mr (Map reduce,
default), tez (hadoop 2 only), spark</description>
+</property>
+
+<property>
+ <name>spark.master</name>
+ <value>local-cluster[2,2,1024]</value>
+</property>
+
+<property>
+ <name>spark.serializer</name>
+ <value>org.apache.spark.serializer.KryoSerializer</value>
+</property>
+
+<property>
+ <name>spark.akka.logLifecycleEvents</name>
+ <value>true</value>
+</property>
+
+<property>
+ <name>hive.spark.log.dir</name>
+ <value>${spark.home}/logs/</value>
+</property>
+
+<property>
+ <name>spark.driver.extraClassPath</name>
+
<value>${maven.local.repository}/org/apache/hive/hive-it-util/${hive.version}/hive-it-util-${hive.version}.jar:${maven.local.repository}/org/apache/hive/hive-exec/${hive.version}/hive-exec-${hive.version}.jar</value>
+</property>
+
+<property>
+ <name>hive.aux.jars.path</name>
+
<value>${maven.local.repository}/org/apache/hive/hive-it-util/${hive.version}/hive-it-util-${hive.version}.jar</value>
+</property>
+
+<property>
+ <name>hive.users.in.admin.role</name>
+ <value>hive_admin_user</value>
+</property>
+
+</configuration>
Added: hive/branches/spark/data/conf/spark/yarn-client/hive-site.xml
URL:
http://svn.apache.org/viewvc/hive/branches/spark/data/conf/spark/yarn-client/hive-site.xml?rev=1656573&view=auto
==============================================================================
--- hive/branches/spark/data/conf/spark/yarn-client/hive-site.xml (added)
+++ hive/branches/spark/data/conf/spark/yarn-client/hive-site.xml Mon Feb 2
21:10:08 2015
@@ -0,0 +1,253 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<configuration>
+
+<!-- Hive Configuration can either be stored in this file or in the hadoop
configuration files -->
+<!-- that are implied by Hadoop setup variables.
-->
+<!-- Aside from Hadoop setup variables - this file is provided as a
convenience so that Hive -->
+<!-- users do not have to edit hadoop configuration files (that may be managed
as a centralized -->
+<!-- resource).
-->
+
+<!-- Hive Execution Parameters -->
+<property>
+ <name>hadoop.tmp.dir</name>
+ <value>${test.tmp.dir}/hadoop-tmp</value>
+ <description>A base for other temporary directories.</description>
+</property>
+
+<property>
+ <name>hive.exec.scratchdir</name>
+ <value>${test.tmp.dir}/scratchdir</value>
+ <description>Scratch space for Hive jobs</description>
+</property>
+
+<property>
+ <name>hive.exec.local.scratchdir</name>
+ <value>${test.tmp.dir}/localscratchdir/</value>
+ <description>Local scratch space for Hive jobs</description>
+</property>
+
+<property>
+ <name>javax.jdo.option.ConnectionURL</name>
+
<value>jdbc:derby:;databaseName=${test.tmp.dir}/junit_metastore_db;create=true</value>
+</property>
+
+<property>
+ <name>hive.stats.dbconnectionstring</name>
+
<value>jdbc:derby:;databaseName=${test.tmp.dir}/TempStatsStore;create=true</value>
+</property>
+
+
+<property>
+ <name>javax.jdo.option.ConnectionDriverName</name>
+ <value>org.apache.derby.jdbc.EmbeddedDriver</value>
+</property>
+
+<property>
+ <name>javax.jdo.option.ConnectionUserName</name>
+ <value>APP</value>
+</property>
+
+<property>
+ <name>javax.jdo.option.ConnectionPassword</name>
+ <value>mine</value>
+</property>
+
+<property>
+ <!-- this should eventually be deprecated since the metastore should supply
this -->
+ <name>hive.metastore.warehouse.dir</name>
+ <value>${test.warehouse.dir}</value>
+ <description></description>
+</property>
+
+<property>
+ <name>hive.metastore.metadb.dir</name>
+ <value>file://${test.tmp.dir}/metadb/</value>
+ <description>
+ Required by metastore server or if the uris argument below is not supplied
+ </description>
+</property>
+
+<property>
+ <name>test.log.dir</name>
+ <value>${test.tmp.dir}/log/</value>
+ <description></description>
+</property>
+
+<property>
+ <name>test.data.files</name>
+ <value>${hive.root}/data/files</value>
+ <description></description>
+</property>
+
+<property>
+ <name>hive.jar.path</name>
+
<value>${maven.local.repository}/org/apache/hive/hive-exec/${hive.version}/hive-exec-${hive.version}.jar</value>
+ <description></description>
+</property>
+
+<property>
+ <name>test.data.scripts</name>
+ <value>${hive.root}/data/scripts</value>
+ <description></description>
+</property>
+
+<property>
+ <name>hive.metastore.rawstore.impl</name>
+ <value>org.apache.hadoop.hive.metastore.ObjectStore</value>
+ <description>Name of the class that implements
org.apache.hadoop.hive.metastore.rawstore interface. This class is used to
store and retrieval of raw metadata objects such as table,
database</description>
+</property>
+
+<property>
+ <name>hive.querylog.location</name>
+ <value>${test.tmp.dir}/tmp</value>
+ <description>Location of the structured hive logs</description>
+</property>
+
+<property>
+ <name>hive.exec.pre.hooks</name>
+ <value>org.apache.hadoop.hive.ql.hooks.PreExecutePrinter,
org.apache.hadoop.hive.ql.hooks.EnforceReadOnlyTables</value>
+ <description>Pre Execute Hook for Tests</description>
+</property>
+
+<property>
+ <name>hive.exec.post.hooks</name>
+ <value>org.apache.hadoop.hive.ql.hooks.PostExecutePrinter</value>
+ <description>Post Execute Hook for Tests</description>
+</property>
+
+<property>
+ <name>hive.support.concurrency</name>
+ <value>false</value>
+ <description>Whether hive supports concurrency or not. A zookeeper instance
must be up and running for the default hive lock manager to support read-write
locks.</description>
+</property>
+
+<property>
+ <name>fs.pfile.impl</name>
+ <value>org.apache.hadoop.fs.ProxyLocalFileSystem</value>
+ <description>A proxy for local file system used for cross file system
testing</description>
+</property>
+
+<property>
+ <name>hive.exec.mode.local.auto</name>
+ <value>false</value>
+ <description>
+ Let hive determine whether to run in local mode automatically
+ Disabling this for tests so that minimr is not affected
+ </description>
+</property>
+
+<property>
+ <name>hive.auto.convert.join</name>
+ <value>false</value>
+ <description>Whether Hive enable the optimization about converting common
join into mapjoin based on the input file size</description>
+</property>
+
+<property>
+ <name>hive.ignore.mapjoin.hint</name>
+ <value>false</value>
+ <description>Whether Hive ignores the mapjoin hint</description>
+</property>
+
+<property>
+ <name>io.sort.mb</name>
+ <value>10</value>
+</property>
+
+<property>
+ <name>hive.input.format</name>
+ <value>org.apache.hadoop.hive.ql.io.CombineHiveInputFormat</value>
+ <description>The default input format, if it is not specified, the system
assigns it. It is set to HiveInputFormat for hadoop versions 17, 18 and 19,
whereas it is set to CombineHiveInputFormat for hadoop 20. The user can always
overwrite it - if there is a bug in CombineHiveInputFormat, it can always be
manually set to HiveInputFormat. </description>
+</property>
+
+<property>
+ <name>hive.default.rcfile.serde</name>
+ <value>org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe</value>
+ <description>The default SerDe hive will use for the rcfile
format</description>
+</property>
+
+<property>
+ <name>hive.stats.dbclass</name>
+ <value>fs</value>
+ <description>The default storatge that stores temporary hive statistics.
Currently, jdbc, hbase and counter type is supported</description>
+</property>
+
+<property>
+ <name>hive.execution.engine</name>
+ <value>spark</value>
+ <description>Chooses execution engine. Options are: mr (Map reduce,
default), tez (hadoop 2 only), spark</description>
+</property>
+
+<property>
+ <name>spark.master</name>
+ <value>yarn-client</value>
+</property>
+
+<property>
+ <name>spark.serializer</name>
+ <value>org.apache.spark.serializer.KryoSerializer</value>
+</property>
+
+<property>
+ <name>spark.akka.logLifecycleEvents</name>
+ <value>true</value>
+</property>
+
+<property>
+ <name>hive.spark.log.dir</name>
+ <value>${spark.home}/logs/</value>
+</property>
+
+<property>
+ <name>spark.driver.extraClassPath</name>
+
<value>${maven.local.repository}/org/apache/hive/hive-it-util/${hive.version}/hive-it-util-${hive.version}.jar:${maven.local.repository}/org/apache/hive/hive-exec/${hive.version}/hive-exec-${hive.version}.jar</value>
+</property>
+
+<property>
+ <name>spark.executor.instances</name>
+ <value>2</value>
+</property>
+
+<property>
+ <name>spark.executor.cores</name>
+ <value>2</value>
+</property>
+
+<property>
+ <name>spark.executor.memory</name>
+ <value>512m</value>
+</property>
+
+<property>
+ <name>spark.testing</name>
+ <value>true</value>
+</property>
+
+<property>
+ <name>hive.aux.jars.path</name>
+
<value>${maven.local.repository}/org/apache/hive/hive-it-util/${hive.version}/hive-it-util-${hive.version}.jar</value>
+</property>
+
+<property>
+ <name>hive.users.in.admin.role</name>
+ <value>hive_admin_user</value>
+</property>
+
+</configuration>
Modified: hive/branches/spark/itests/pom.xml
URL:
http://svn.apache.org/viewvc/hive/branches/spark/itests/pom.xml?rev=1656573&r1=1656572&r2=1656573&view=diff
==============================================================================
--- hive/branches/spark/itests/pom.xml (original)
+++ hive/branches/spark/itests/pom.xml Mon Feb 2 21:10:08 2015
@@ -87,7 +87,7 @@
curl -Sso $DOWNLOAD_DIR/$tarName $url
fi
tar -zxf $DOWNLOAD_DIR/$tarName -C $BASE_DIR
- mv $BASE_DIR/${finalName}* $BASE_DIR/$finalName
+ mv
$BASE_DIR/spark-${spark.version}-bin-hadoop2-without-hive $BASE_DIR/$finalName
}
mkdir -p $DOWNLOAD_DIR
download
"http://d3jw87u4immizc.cloudfront.net/spark-tarball/spark-${spark.version}-bin-hadoop2-without-hive.tgz";
"spark"
Modified: hive/branches/spark/itests/qtest-spark/pom.xml
URL:
http://svn.apache.org/viewvc/hive/branches/spark/itests/qtest-spark/pom.xml?rev=1656573&r1=1656572&r2=1656573&view=diff
==============================================================================
--- hive/branches/spark/itests/qtest-spark/pom.xml (original)
+++ hive/branches/spark/itests/qtest-spark/pom.xml Mon Feb 2 21:10:08 2015
@@ -37,7 +37,6 @@
<qfile></qfile>
<qfile_regex></qfile_regex>
<run_disabled>false</run_disabled>
- <clustermode>spark</clustermode>
<execute.beeline.tests>false</execute.beeline.tests>
<active.hadoop.version>${hadoop-23.version}</active.hadoop.version>
<test.dfs.mkdir>-mkdir -p</test.dfs.mkdir>
@@ -333,22 +332,39 @@
<taskdef name="qtestgen"
classname="org.apache.hadoop.hive.ant.QTestGenTask"
classpath="${test.classpath}" />
<mkdir
dir="${project.build.directory}/qfile-results/clientpositive/spark" />
+ <mkdir
dir="${project.build.directory}/qfile-results/clientpositive/miniSparkOnYarn" />
<qtestgen hiveRootDirectory="${basedir}/${hive.path.to.root}/"
outputDirectory="${project.build.directory}/generated-test-sources/java/org/apache/hadoop/hive/cli/"
templatePath="${basedir}/${hive.path.to.root}/ql/src/test/templates/"
template="TestCliDriver.vm"
queryDirectory="${basedir}/${hive.path.to.root}/ql/src/test/queries/clientpositive/"
queryFile="${qfile}"
queryFileRegex="${qfile_regex}"
- clusterMode="${clustermode}"
+ clusterMode="spark"
includeQueryFile="${spark.query.files}"
runDisabled="${run_disabled}"
- hiveConfDir="${basedir}/${hive.path.to.root}/data/conf/spark"
+
hiveConfDir="${basedir}/${hive.path.to.root}/data/conf/spark/standalone"
resultsDirectory="${basedir}/${hive.path.to.root}/ql/src/test/results/clientpositive/spark"
className="TestSparkCliDriver"
logFile="${project.build.directory}/testsparkclidrivergen.log"
logDirectory="${project.build.directory}/qfile-results/clientpositive/spark"
initScript="q_test_init.sql"
cleanupScript="q_test_cleanup.sql"/>
+ <qtestgen hiveRootDirectory="${basedir}/${hive.path.to.root}/"
+
outputDirectory="${project.build.directory}/generated-test-sources/java/org/apache/hadoop/hive/cli/"
+
templatePath="${basedir}/${hive.path.to.root}/ql/src/test/templates/"
template="TestCliDriver.vm"
+
queryDirectory="${basedir}/${hive.path.to.root}/ql/src/test/queries/clientpositive/"
+ queryFile="${qfile}"
+ queryFileRegex="${qfile_regex}"
+ clusterMode="miniSparkOnYarn"
+ includeQueryFile="${miniSparkOnYarn.query.files}"
+ runDisabled="${run_disabled}"
+
hiveConfDir="${basedir}/${hive.path.to.root}/data/conf/spark/yarn-client"
+
resultsDirectory="${basedir}/${hive.path.to.root}/ql/src/test/results/clientpositive/spark"
+ className="TestMiniSparkOnYarnCliDriver"
+
logFile="${project.build.directory}/testminisparkonyarnclidrivergen.log"
+
logDirectory="${project.build.directory}/qfile-results/clientpositive/spark"
+ initScript="q_test_init.sql"
+ cleanupScript="q_test_cleanup.sql"/>
</target>
</configuration>
<goals>
Modified:
hive/branches/spark/itests/src/test/resources/testconfiguration.properties
URL:
http://svn.apache.org/viewvc/hive/branches/spark/itests/src/test/resources/testconfiguration.properties?rev=1656573&r1=1656572&r2=1656573&view=diff
==============================================================================
--- hive/branches/spark/itests/src/test/resources/testconfiguration.properties
(original)
+++ hive/branches/spark/itests/src/test/resources/testconfiguration.properties
Mon Feb 2 21:10:08 2015
@@ -713,6 +713,7 @@ spark.query.files=add_part_multiple.q, \
join_thrift.q, \
join_vc.q, \
join_view.q, \
+ lateral_view_explode2.q, \
leftsemijoin.q, \
leftsemijoin_mr.q, \
limit_partition_metadataonly.q, \
@@ -996,5 +997,48 @@ spark.query.files=add_part_multiple.q, \
vectorized_shufflejoin.q, \
vectorized_string_funcs.q, \
vectorized_timestamp_funcs.q, \
- windowing.q, \
- lateral_view_explode2.q
+ windowing.q
+
+miniSparkOnYarn.query.files=auto_sortmerge_join_16.q,\
+ bucket4.q,\
+ bucket5.q,\
+ bucket6.q,\
+ bucketizedhiveinputformat.q,\
+ bucketmapjoin6.q,\
+ bucketmapjoin7.q,\
+ constprog_partitioner.q,\
+ disable_merge_for_bucketing.q,\
+ empty_dir_in_table.q,\
+ external_table_with_space_in_location_path.q,\
+ file_with_header_footer.q,\
+ import_exported_table.q,\
+ index_bitmap3.q,\
+ index_bitmap_auto.q,\
+ infer_bucket_sort_bucketed_table.q,\
+ infer_bucket_sort_map_operators.q,\
+ infer_bucket_sort_merge.q,\
+ infer_bucket_sort_num_buckets.q,\
+ infer_bucket_sort_reducers_power_two.q,\
+ input16_cc.q,\
+ leftsemijoin_mr.q,\
+ list_bucket_dml_10.q,\
+ load_fs2.q,\
+ load_hdfs_file_with_space_in_the_name.q,\
+ optrstat_groupby.q,\
+ parallel_orderby.q,\
+ ql_rewrite_gbtoidx.q,\
+ ql_rewrite_gbtoidx_cbo_1.q,\
+ quotedid_smb.q,\
+ reduce_deduplicate.q,\
+ remote_script.q,\
+ root_dir_external_table.q,\
+ schemeAuthority.q,\
+ schemeAuthority2.q,\
+ scriptfile1.q,\
+ scriptfile1_win.q,\
+ smb_mapjoin_8.q,\
+ stats_counter.q,\
+ stats_counter_partitioned.q,\
+ temp_table_external.q,\
+ truncate_column_buckets.q,\
+ uber_reduce.q
Modified:
hive/branches/spark/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
URL:
http://svn.apache.org/viewvc/hive/branches/spark/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java?rev=1656573&r1=1656572&r2=1656573&view=diff
==============================================================================
---
hive/branches/spark/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
(original)
+++
hive/branches/spark/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
Mon Feb 2 21:10:08 2015
@@ -302,6 +302,7 @@ public class QTestUtil {
tez,
spark,
encrypted,
+ miniSparkOnYarn,
none;
public static MiniClusterType valueForString(String type) {
@@ -313,6 +314,8 @@ public class QTestUtil {
return spark;
} else if (type.equals("encrypted")) {
return encrypted;
+ } else if (type.equals("miniSparkOnYarn")) {
+ return miniSparkOnYarn;
} else {
return none;
}
@@ -380,6 +383,8 @@ public class QTestUtil {
String uriString =
WindowsPathUtil.getHdfsUriString(fs.getUri().toString());
if (clusterType == MiniClusterType.tez) {
mr = shims.getMiniTezCluster(conf, 4, uriString, 1);
+ } else if (clusterType == MiniClusterType.miniSparkOnYarn) {
+ mr = shims.getMiniSparkCluster(conf, 4, uriString, 1);
} else {
mr = shims.getMiniMrCluster(conf, 4, uriString, 1);
}
@@ -875,7 +880,8 @@ public class QTestUtil {
ss.setIsSilent(true);
SessionState oldSs = SessionState.get();
- if (oldSs != null && (clusterType == MiniClusterType.tez || clusterType ==
MiniClusterType.spark)) {
+ if (oldSs != null && (clusterType == MiniClusterType.tez || clusterType ==
MiniClusterType.spark
+ || clusterType == MiniClusterType.miniSparkOnYarn)) {
sparkSession = oldSs.getSparkSession();
ss.setSparkSession(sparkSession);
oldSs.setSparkSession(null);
@@ -935,7 +941,8 @@ public class QTestUtil {
ss.err = System.out;
SessionState oldSs = SessionState.get();
- if (oldSs != null && (clusterType == MiniClusterType.tez || clusterType ==
MiniClusterType.spark)) {
+ if (oldSs != null && (clusterType == MiniClusterType.tez || clusterType ==
MiniClusterType.spark
+ || clusterType == MiniClusterType.miniSparkOnYarn)) {
sparkSession = oldSs.getSparkSession();
ss.setSparkSession(sparkSession);
oldSs.setSparkSession(null);
Modified:
hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/RemoteHiveSparkClient.java
URL:
http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/RemoteHiveSparkClient.java?rev=1656573&r1=1656572&r2=1656573&view=diff
==============================================================================
---
hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/RemoteHiveSparkClient.java
(original)
+++
hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/RemoteHiveSparkClient.java
Mon Feb 2 21:10:08 2015
@@ -23,6 +23,7 @@ import com.google.common.base.Strings;
import java.io.IOException;
import java.io.Serializable;
import java.net.MalformedURLException;
+import java.net.URL;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
@@ -76,8 +77,8 @@ public class RemoteHiveSparkClient imple
private transient SparkConf sparkConf;
private transient HiveConf hiveConf;
- private transient List<String> localJars = new ArrayList<String>();
- private transient List<String> localFiles = new ArrayList<String>();
+ private transient List<URL> localJars = new ArrayList<URL>();
+ private transient List<URL> localFiles = new ArrayList<URL>();
private final transient long sparkClientTimtout;
@@ -161,26 +162,28 @@ public class RemoteHiveSparkClient imple
private void addResources(String addedFiles) {
for (String addedFile :
CSV_SPLITTER.split(Strings.nullToEmpty(addedFiles))) {
- if (!localFiles.contains(addedFile)) {
- localFiles.add(addedFile);
- try {
- remoteClient.addFile(SparkUtilities.getURL(addedFile));
- } catch (MalformedURLException e) {
- LOG.warn("Failed to add file:" + addedFile);
+ try {
+ URL fileUrl = SparkUtilities.getURL(addedFile);
+ if (fileUrl != null && !localFiles.contains(fileUrl)) {
+ localFiles.add(fileUrl);
+ remoteClient.addFile(fileUrl);
}
+ } catch (MalformedURLException e) {
+ LOG.warn("Failed to add file:" + addedFile);
}
}
}
private void addJars(String addedJars) {
for (String addedJar : CSV_SPLITTER.split(Strings.nullToEmpty(addedJars)))
{
- if (!localJars.contains(addedJar)) {
- localJars.add(addedJar);
- try {
- remoteClient.addJar(SparkUtilities.getURL(addedJar));
- } catch (MalformedURLException e) {
- LOG.warn("Failed to add jar:" + addedJar);
+ try {
+ URL jarUrl = SparkUtilities.getURL(addedJar);
+ if (jarUrl != null && !localJars.contains(jarUrl)) {
+ localJars.add(jarUrl);
+ remoteClient.addJar(jarUrl);
}
+ } catch (MalformedURLException e) {
+ LOG.warn("Failed to add jar:" + addedJar);
}
}
}
Added:
hive/branches/spark/ql/src/test/results/clientpositive/spark/bucket5.q.out
URL:
http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/bucket5.q.out?rev=1656573&view=auto
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/spark/bucket5.q.out
(added)
+++ hive/branches/spark/ql/src/test/results/clientpositive/spark/bucket5.q.out
Mon Feb 2 21:10:08 2015
@@ -0,0 +1,455 @@
+PREHOOK: query: -- Tests that when a multi insert inserts into a bucketed
table and a table which is not bucketed
+-- the bucketed table is not merged and the table which is not bucketed is
+
+CREATE TABLE bucketed_table(key INT, value STRING) CLUSTERED BY (key) SORTED
BY (key) INTO 2 BUCKETS
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@bucketed_table
+POSTHOOK: query: -- Tests that when a multi insert inserts into a bucketed
table and a table which is not bucketed
+-- the bucketed table is not merged and the table which is not bucketed is
+
+CREATE TABLE bucketed_table(key INT, value STRING) CLUSTERED BY (key) SORTED
BY (key) INTO 2 BUCKETS
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@bucketed_table
+PREHOOK: query: CREATE TABLE unbucketed_table(key INT, value STRING)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@unbucketed_table
+POSTHOOK: query: CREATE TABLE unbucketed_table(key INT, value STRING)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@unbucketed_table
+PREHOOK: query: EXPLAIN EXTENDED
+FROM src
+INSERT OVERWRITE TABLE bucketed_table SELECT key, value
+INSERT OVERWRITE TABLE unbucketed_table SELECT key, value cluster by key
+PREHOOK: type: QUERY
+POSTHOOK: query: EXPLAIN EXTENDED
+FROM src
+INSERT OVERWRITE TABLE bucketed_table SELECT key, value
+INSERT OVERWRITE TABLE unbucketed_table SELECT key, value cluster by key
+POSTHOOK: type: QUERY
+ABSTRACT SYNTAX TREE:
+
+TOK_QUERY
+ TOK_FROM
+ TOK_TABREF
+ TOK_TABNAME
+ src
+ TOK_INSERT
+ TOK_DESTINATION
+ TOK_TAB
+ TOK_TABNAME
+ bucketed_table
+ TOK_SELECT
+ TOK_SELEXPR
+ TOK_TABLE_OR_COL
+ key
+ TOK_SELEXPR
+ TOK_TABLE_OR_COL
+ value
+ TOK_INSERT
+ TOK_DESTINATION
+ TOK_TAB
+ TOK_TABNAME
+ unbucketed_table
+ TOK_SELECT
+ TOK_SELEXPR
+ TOK_TABLE_OR_COL
+ key
+ TOK_SELEXPR
+ TOK_TABLE_OR_COL
+ value
+ TOK_CLUSTERBY
+ TOK_TABLE_OR_COL
+ key
+
+
+STAGE DEPENDENCIES:
+ Stage-2 is a root stage
+ Stage-0 depends on stages: Stage-2
+ Stage-3 depends on stages: Stage-0
+ Stage-1 depends on stages: Stage-2
+ Stage-4 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-2
+ Spark
+ Edges:
+ Reducer 2 <- Map 4 (PARTITION-LEVEL SORT, 2)
+ Reducer 3 <- Map 5 (SORT, 2)
+#### A masked pattern was here ####
+ Vertices:
+ Map 4
+ Map Operator Tree:
+ TableScan
+ alias: src
+ Statistics: Num rows: 500 Data size: 5312 Basic stats:
COMPLETE Column stats: NONE
+ GatherStats: false
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 500 Data size: 5312 Basic stats:
COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: UDFToInteger(_col0) (type: int)
+ sort order: +
+ Map-reduce partition columns: UDFToInteger(_col0) (type:
int)
+ Statistics: Num rows: 500 Data size: 5312 Basic stats:
COMPLETE Column stats: NONE
+ tag: -1
+ value expressions: _col0 (type: string), _col1 (type:
string)
+ auto parallelism: false
+ Path -> Alias:
+#### A masked pattern was here ####
+ Path -> Partition:
+#### A masked pattern was here ####
+ Partition
+ base file name: src
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format:
org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ COLUMN_STATS_ACCURATE true
+ bucket_count -1
+ columns key,value
+ columns.comments 'default','default'
+ columns.types string:string
+#### A masked pattern was here ####
+ name default.src
+ numFiles 1
+ numRows 500
+ rawDataSize 5312
+ serialization.ddl struct src { string key, string value}
+ serialization.format 1
+ serialization.lib
org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ totalSize 5812
+#### A masked pattern was here ####
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format:
org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ COLUMN_STATS_ACCURATE true
+ bucket_count -1
+ columns key,value
+ columns.comments 'default','default'
+ columns.types string:string
+#### A masked pattern was here ####
+ name default.src
+ numFiles 1
+ numRows 500
+ rawDataSize 5312
+ serialization.ddl struct src { string key, string value}
+ serialization.format 1
+ serialization.lib
org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ totalSize 5812
+#### A masked pattern was here ####
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.src
+ name: default.src
+ Truncated Path -> Alias:
+ /src [src]
+ Map 5
+ Map Operator Tree:
+ TableScan
+ alias: src
+ Statistics: Num rows: 500 Data size: 5312 Basic stats:
COMPLETE Column stats: NONE
+ GatherStats: false
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 500 Data size: 5312 Basic stats:
COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 500 Data size: 5312 Basic stats:
COMPLETE Column stats: NONE
+ tag: -1
+ value expressions: _col1 (type: string)
+ auto parallelism: false
+ Path -> Alias:
+#### A masked pattern was here ####
+ Path -> Partition:
+#### A masked pattern was here ####
+ Partition
+ base file name: src
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format:
org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ COLUMN_STATS_ACCURATE true
+ bucket_count -1
+ columns key,value
+ columns.comments 'default','default'
+ columns.types string:string
+#### A masked pattern was here ####
+ name default.src
+ numFiles 1
+ numRows 500
+ rawDataSize 5312
+ serialization.ddl struct src { string key, string value}
+ serialization.format 1
+ serialization.lib
org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ totalSize 5812
+#### A masked pattern was here ####
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format:
org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ COLUMN_STATS_ACCURATE true
+ bucket_count -1
+ columns key,value
+ columns.comments 'default','default'
+ columns.types string:string
+#### A masked pattern was here ####
+ name default.src
+ numFiles 1
+ numRows 500
+ rawDataSize 5312
+ serialization.ddl struct src { string key, string value}
+ serialization.format 1
+ serialization.lib
org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ totalSize 5812
+#### A masked pattern was here ####
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.src
+ name: default.src
+ Truncated Path -> Alias:
+ /src [src]
+ Reducer 2
+ Needs Tagging: false
+ Reduce Operator Tree:
+ Extract
+ Statistics: Num rows: 500 Data size: 5312 Basic stats:
COMPLETE Column stats: NONE
+ Select Operator
+ expressions: UDFToInteger(_col0) (type: int), _col1 (type:
string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 500 Data size: 5312 Basic stats:
COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ GlobalTableId: 1
+#### A masked pattern was here ####
+ NumFilesPerFileSink: 1
+ Statistics: Num rows: 500 Data size: 5312 Basic stats:
COMPLETE Column stats: NONE
+#### A masked pattern was here ####
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format:
org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ SORTBUCKETCOLSPREFIX TRUE
+ bucket_count 2
+ bucket_field_name key
+ columns key,value
+ columns.comments
+ columns.types int:string
+#### A masked pattern was here ####
+ name default.bucketed_table
+ serialization.ddl struct bucketed_table { i32 key,
string value}
+ serialization.format 1
+ serialization.lib
org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+#### A masked pattern was here ####
+ serde:
org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.bucketed_table
+ TotalFiles: 1
+ GatherStats: true
+ MultiFileSpray: false
+ Reducer 3
+ Needs Tagging: false
+ Reduce Operator Tree:
+ Select Operator
+ expressions: UDFToInteger(KEY.reducesinkkey0) (type: int),
VALUE._col0 (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 500 Data size: 5312 Basic stats:
COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ GlobalTableId: 2
+#### A masked pattern was here ####
+ NumFilesPerFileSink: 1
+ Statistics: Num rows: 500 Data size: 5312 Basic stats:
COMPLETE Column stats: NONE
+#### A masked pattern was here ####
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format:
org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ bucket_count -1
+ columns key,value
+ columns.comments
+ columns.types int:string
+#### A masked pattern was here ####
+ name default.unbucketed_table
+ serialization.ddl struct unbucketed_table { i32 key,
string value}
+ serialization.format 1
+ serialization.lib
org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+#### A masked pattern was here ####
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.unbucketed_table
+ TotalFiles: 1
+ GatherStats: true
+ MultiFileSpray: false
+
+ Stage: Stage-0
+ Move Operator
+ tables:
+ replace: true
+#### A masked pattern was here ####
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format:
org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ SORTBUCKETCOLSPREFIX TRUE
+ bucket_count 2
+ bucket_field_name key
+ columns key,value
+ columns.comments
+ columns.types int:string
+#### A masked pattern was here ####
+ name default.bucketed_table
+ serialization.ddl struct bucketed_table { i32 key, string
value}
+ serialization.format 1
+ serialization.lib
org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+#### A masked pattern was here ####
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.bucketed_table
+
+ Stage: Stage-3
+ Stats-Aggr Operator
+#### A masked pattern was here ####
+
+ Stage: Stage-1
+ Move Operator
+ tables:
+ replace: true
+#### A masked pattern was here ####
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format:
org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ bucket_count -1
+ columns key,value
+ columns.comments
+ columns.types int:string
+#### A masked pattern was here ####
+ name default.unbucketed_table
+ serialization.ddl struct unbucketed_table { i32 key, string
value}
+ serialization.format 1
+ serialization.lib
org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+#### A masked pattern was here ####
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.unbucketed_table
+
+ Stage: Stage-4
+ Stats-Aggr Operator
+#### A masked pattern was here ####
+
+PREHOOK: query: FROM src
+INSERT OVERWRITE TABLE bucketed_table SELECT key, value
+INSERT OVERWRITE TABLE unbucketed_table SELECT key, value cluster by key
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@bucketed_table
+PREHOOK: Output: default@unbucketed_table
+POSTHOOK: query: FROM src
+INSERT OVERWRITE TABLE bucketed_table SELECT key, value
+INSERT OVERWRITE TABLE unbucketed_table SELECT key, value cluster by key
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@bucketed_table
+POSTHOOK: Output: default@unbucketed_table
+POSTHOOK: Lineage: bucketed_table.key EXPRESSION
[(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: bucketed_table.value SIMPLE
[(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: unbucketed_table.key EXPRESSION
[(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: unbucketed_table.value SIMPLE
[(src)src.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: DESC FORMATTED bucketed_table
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@bucketed_table
+POSTHOOK: query: DESC FORMATTED bucketed_table
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@bucketed_table
+# col_name data_type comment
+
+key int
+value string
+
+# Detailed Table Information
+Database: default
+#### A masked pattern was here ####
+Protect Mode: None
+Retention: 0
+#### A masked pattern was here ####
+Table Type: MANAGED_TABLE
+Table Parameters:
+ COLUMN_STATS_ACCURATE true
+ SORTBUCKETCOLSPREFIX TRUE
+ numFiles 2
+ numRows 500
+ rawDataSize 5312
+ totalSize 5812
+#### A masked pattern was here ####
+
+# Storage Information
+SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+InputFormat: org.apache.hadoop.mapred.TextInputFormat
+OutputFormat:
org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+Compressed: No
+Num Buckets: 2
+Bucket Columns: [key]
+Sort Columns: [Order(col:key, order:1)]
+Storage Desc Params:
+ serialization.format 1
+PREHOOK: query: SELECT * FROM bucketed_table TABLESAMPLE (BUCKET 1 OUT OF 2) s
LIMIT 10
+PREHOOK: type: QUERY
+PREHOOK: Input: default@bucketed_table
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT * FROM bucketed_table TABLESAMPLE (BUCKET 1 OUT OF 2)
s LIMIT 10
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@bucketed_table
+#### A masked pattern was here ####
+0 val_0
+0 val_0
+0 val_0
+2 val_2
+4 val_4
+8 val_8
+10 val_10
+12 val_12
+12 val_12
+18 val_18
+PREHOOK: query: SELECT * FROM bucketed_table TABLESAMPLE (BUCKET 2 OUT OF 2) s
LIMIT 10
+PREHOOK: type: QUERY
+PREHOOK: Input: default@bucketed_table
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT * FROM bucketed_table TABLESAMPLE (BUCKET 2 OUT OF 2)
s LIMIT 10
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@bucketed_table
+#### A masked pattern was here ####
+5 val_5
+5 val_5
+5 val_5
+9 val_9
+11 val_11
+15 val_15
+15 val_15
+17 val_17
+19 val_19
+27 val_27
+PREHOOK: query: -- Should be 2 (not merged)
+SELECT COUNT(DISTINCT INPUT__FILE__NAME) FROM bucketed_table
+PREHOOK: type: QUERY
+PREHOOK: Input: default@bucketed_table
+#### A masked pattern was here ####
+POSTHOOK: query: -- Should be 2 (not merged)
+SELECT COUNT(DISTINCT INPUT__FILE__NAME) FROM bucketed_table
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@bucketed_table
+#### A masked pattern was here ####
+2
+PREHOOK: query: -- Should be 1 (merged)
+SELECT COUNT(DISTINCT INPUT__FILE__NAME) FROM unbucketed_table
+PREHOOK: type: QUERY
+PREHOOK: Input: default@unbucketed_table
+#### A masked pattern was here ####
+POSTHOOK: query: -- Should be 1 (merged)
+SELECT COUNT(DISTINCT INPUT__FILE__NAME) FROM unbucketed_table
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@unbucketed_table
+#### A masked pattern was here ####
+2
Added:
hive/branches/spark/ql/src/test/results/clientpositive/spark/bucket6.q.out
URL:
http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/bucket6.q.out?rev=1656573&view=auto
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/spark/bucket6.q.out
(added)
+++ hive/branches/spark/ql/src/test/results/clientpositive/spark/bucket6.q.out
Mon Feb 2 21:10:08 2015
@@ -0,0 +1,193 @@
+PREHOOK: query: CREATE TABLE src_bucket(key STRING, value STRING) CLUSTERED BY
(key) SORTED BY (key) INTO 2 BUCKETS
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@src_bucket
+POSTHOOK: query: CREATE TABLE src_bucket(key STRING, value STRING) CLUSTERED
BY (key) SORTED BY (key) INTO 2 BUCKETS
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@src_bucket
+PREHOOK: query: explain
+insert into table src_bucket select key,value from srcpart
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+insert into table src_bucket select key,value from srcpart
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+ Stage-2 depends on stages: Stage-0
+
+STAGE PLANS:
+ Stage: Stage-1
+ Spark
+ Edges:
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: srcpart
+ Statistics: Num rows: 2000 Data size: 21248 Basic stats:
COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 2000 Data size: 21248 Basic stats:
COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 2000 Data size: 21248 Basic stats:
COMPLETE Column stats: NONE
+ value expressions: _col0 (type: string), _col1 (type:
string)
+ Reducer 2
+ Reduce Operator Tree:
+ Extract
+ Statistics: Num rows: 2000 Data size: 21248 Basic stats:
COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 2000 Data size: 21248 Basic stats:
COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format:
org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.src_bucket
+
+ Stage: Stage-0
+ Move Operator
+ tables:
+ replace: false
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format:
org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.src_bucket
+
+ Stage: Stage-2
+ Stats-Aggr Operator
+
+PREHOOK: query: insert into table src_bucket select key,value from srcpart
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcpart
+PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
+PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
+PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11
+PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12
+PREHOOK: Output: default@src_bucket
+POSTHOOK: query: insert into table src_bucket select key,value from srcpart
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcpart
+POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
+POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
+POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11
+POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12
+POSTHOOK: Output: default@src_bucket
+POSTHOOK: Lineage: src_bucket.key SIMPLE
[(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: src_bucket.value SIMPLE
[(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: select * from src_bucket limit 100
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src_bucket
+#### A masked pattern was here ####
+POSTHOOK: query: select * from src_bucket limit 100
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src_bucket
+#### A masked pattern was here ####
+0 val_0
+0 val_0
+0 val_0
+0 val_0
+0 val_0
+0 val_0
+0 val_0
+0 val_0
+0 val_0
+0 val_0
+0 val_0
+0 val_0
+103 val_103
+103 val_103
+103 val_103
+103 val_103
+103 val_103
+103 val_103
+103 val_103
+103 val_103
+105 val_105
+105 val_105
+105 val_105
+105 val_105
+11 val_11
+11 val_11
+11 val_11
+11 val_11
+114 val_114
+114 val_114
+114 val_114
+114 val_114
+116 val_116
+116 val_116
+116 val_116
+116 val_116
+118 val_118
+118 val_118
+118 val_118
+118 val_118
+118 val_118
+118 val_118
+118 val_118
+118 val_118
+125 val_125
+125 val_125
+125 val_125
+125 val_125
+125 val_125
+125 val_125
+125 val_125
+125 val_125
+129 val_129
+129 val_129
+129 val_129
+129 val_129
+129 val_129
+129 val_129
+129 val_129
+129 val_129
+134 val_134
+134 val_134
+134 val_134
+134 val_134
+134 val_134
+134 val_134
+134 val_134
+134 val_134
+136 val_136
+136 val_136
+136 val_136
+136 val_136
+138 val_138
+138 val_138
+138 val_138
+138 val_138
+138 val_138
+138 val_138
+138 val_138
+138 val_138
+138 val_138
+138 val_138
+138 val_138
+138 val_138
+138 val_138
+138 val_138
+138 val_138
+138 val_138
+143 val_143
+143 val_143
+143 val_143
+143 val_143
+145 val_145
+145 val_145
+145 val_145
+145 val_145
+149 val_149
+149 val_149
+149 val_149
+149 val_149
Added:
hive/branches/spark/ql/src/test/results/clientpositive/spark/bucketizedhiveinputformat.q.out
URL:
http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/bucketizedhiveinputformat.q.out?rev=1656573&view=auto
==============================================================================
---
hive/branches/spark/ql/src/test/results/clientpositive/spark/bucketizedhiveinputformat.q.out
(added)
+++
hive/branches/spark/ql/src/test/results/clientpositive/spark/bucketizedhiveinputformat.q.out
Mon Feb 2 21:10:08 2015
@@ -0,0 +1,111 @@
+PREHOOK: query: CREATE TABLE T1(name STRING) STORED AS TEXTFILE
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@T1
+POSTHOOK: query: CREATE TABLE T1(name STRING) STORED AS TEXTFILE
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@T1
+PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/kv1.txt' INTO TABLE T1
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@t1
+POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/kv1.txt' INTO TABLE
T1
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@t1
+PREHOOK: query: CREATE TABLE T2(name STRING) STORED AS SEQUENCEFILE
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@T2
+POSTHOOK: query: CREATE TABLE T2(name STRING) STORED AS SEQUENCEFILE
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@T2
+PREHOOK: query: INSERT OVERWRITE TABLE T2 SELECT * FROM (
+SELECT tmp1.name as name FROM (
+ SELECT name, 'MMM' AS n FROM T1) tmp1
+ JOIN (SELECT 'MMM' AS n FROM T1) tmp2
+ JOIN (SELECT 'MMM' AS n FROM T1) tmp3
+ ON tmp1.n = tmp2.n AND tmp1.n = tmp3.n) ttt LIMIT 5000000
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t1
+PREHOOK: Output: default@t2
+POSTHOOK: query: INSERT OVERWRITE TABLE T2 SELECT * FROM (
+SELECT tmp1.name as name FROM (
+ SELECT name, 'MMM' AS n FROM T1) tmp1
+ JOIN (SELECT 'MMM' AS n FROM T1) tmp2
+ JOIN (SELECT 'MMM' AS n FROM T1) tmp3
+ ON tmp1.n = tmp2.n AND tmp1.n = tmp3.n) ttt LIMIT 5000000
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@t1
+POSTHOOK: Output: default@t2
+POSTHOOK: Lineage: t2.name SIMPLE [(t1)t1.FieldSchema(name:name, type:string,
comment:null), ]
+PREHOOK: query: CREATE TABLE T3(name STRING) STORED AS TEXTFILE
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@T3
+POSTHOOK: query: CREATE TABLE T3(name STRING) STORED AS TEXTFILE
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@T3
+PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/kv1.txt' INTO TABLE T3
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@t3
+POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/kv1.txt' INTO TABLE
T3
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@t3
+PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/kv2.txt' INTO TABLE T3
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@t3
+POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/kv2.txt' INTO TABLE
T3
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@t3
+PREHOOK: query: -- 2 split by max.split.size
+SELECT COUNT(1) FROM T2
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t2
+#### A masked pattern was here ####
+POSTHOOK: query: -- 2 split by max.split.size
+SELECT COUNT(1) FROM T2
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@t2
+#### A masked pattern was here ####
+5000000
+PREHOOK: query: -- 1 split for two file
+SELECT COUNT(1) FROM T3
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t3
+#### A masked pattern was here ####
+POSTHOOK: query: -- 1 split for two file
+SELECT COUNT(1) FROM T3
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@t3
+#### A masked pattern was here ####
+1000
+PREHOOK: query: -- 1 split
+SELECT COUNT(1) FROM T2
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t2
+#### A masked pattern was here ####
+POSTHOOK: query: -- 1 split
+SELECT COUNT(1) FROM T2
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@t2
+#### A masked pattern was here ####
+5000000
+PREHOOK: query: -- 2 split for two file
+SELECT COUNT(1) FROM T3
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t3
+#### A masked pattern was here ####
+POSTHOOK: query: -- 2 split for two file
+SELECT COUNT(1) FROM T3
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@t3
+#### A masked pattern was here ####
+1000
Added:
hive/branches/spark/ql/src/test/results/clientpositive/spark/constprog_partitioner.q.out
URL:
http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/constprog_partitioner.q.out?rev=1656573&view=auto
==============================================================================
---
hive/branches/spark/ql/src/test/results/clientpositive/spark/constprog_partitioner.q.out
(added)
+++
hive/branches/spark/ql/src/test/results/clientpositive/spark/constprog_partitioner.q.out
Mon Feb 2 21:10:08 2015
@@ -0,0 +1,191 @@
+PREHOOK: query: EXPLAIN
+SELECT src1.key, src1.key + 1, src2.value
+ FROM src src1 join src src2 ON src1.key = src2.key AND src1.key = 100
+PREHOOK: type: QUERY
+POSTHOOK: query: EXPLAIN
+SELECT src1.key, src1.key + 1, src2.value
+ FROM src src1 join src src2 ON src1.key = src2.key AND src1.key = 100
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Spark
+ Edges:
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 4), Map 3 (PARTITION-LEVEL
SORT, 4)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: src1
+ Statistics: Num rows: 500 Data size: 5312 Basic stats:
COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: (key = 100) (type: boolean)
+ Statistics: Num rows: 250 Data size: 2656 Basic stats:
COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: '100' (type: string)
+ sort order: +
+ Map-reduce partition columns: '100' (type: string)
+ Statistics: Num rows: 250 Data size: 2656 Basic stats:
COMPLETE Column stats: NONE
+ Map 3
+ Map Operator Tree:
+ TableScan
+ alias: src2
+ Statistics: Num rows: 500 Data size: 5312 Basic stats:
COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: (key = 100) (type: boolean)
+ Statistics: Num rows: 250 Data size: 2656 Basic stats:
COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: '100' (type: string)
+ sort order: +
+ Map-reduce partition columns: '100' (type: string)
+ Statistics: Num rows: 250 Data size: 2656 Basic stats:
COMPLETE Column stats: NONE
+ value expressions: value (type: string)
+ Reducer 2
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 key (type: string)
+ 1 key (type: string)
+ outputColumnNames: _col6
+ Statistics: Num rows: 275 Data size: 2921 Basic stats:
COMPLETE Column stats: NONE
+ Select Operator
+ expressions: '100' (type: string), 101.0 (type: double),
_col6 (type: string)
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 275 Data size: 2921 Basic stats:
COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 275 Data size: 2921 Basic stats:
COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format:
org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde:
org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: SELECT src1.key, src1.key + 1, src2.value
+ FROM src src1 join src src2 ON src1.key = src2.key AND src1.key = 100
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT src1.key, src1.key + 1, src2.value
+ FROM src src1 join src src2 ON src1.key = src2.key AND src1.key = 100
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+100 101.0 val_100
+100 101.0 val_100
+100 101.0 val_100
+100 101.0 val_100
+PREHOOK: query: EXPLAIN
+SELECT l_partkey, l_suppkey
+FROM lineitem li
+WHERE li.l_linenumber = 1 AND
+ li.l_orderkey IN (SELECT l_orderkey FROM lineitem WHERE l_shipmode = 'AIR'
AND l_linenumber = li.l_linenumber)
+PREHOOK: type: QUERY
+POSTHOOK: query: EXPLAIN
+SELECT l_partkey, l_suppkey
+FROM lineitem li
+WHERE li.l_linenumber = 1 AND
+ li.l_orderkey IN (SELECT l_orderkey FROM lineitem WHERE l_shipmode = 'AIR'
AND l_linenumber = li.l_linenumber)
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Spark
+ Edges:
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 4), Map 3 (PARTITION-LEVEL
SORT, 4)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: li
+ Statistics: Num rows: 100 Data size: 11999 Basic stats:
COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: ((l_orderkey is not null and l_linenumber is
not null) and (l_linenumber = 1)) (type: boolean)
+ Statistics: Num rows: 12 Data size: 1439 Basic stats:
COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: l_orderkey (type: int), l_linenumber
(type: int)
+ sort order: ++
+ Map-reduce partition columns: l_orderkey (type: int),
l_linenumber (type: int)
+ Statistics: Num rows: 12 Data size: 1439 Basic stats:
COMPLETE Column stats: NONE
+ value expressions: l_partkey (type: int), l_suppkey
(type: int)
+ Map 3
+ Map Operator Tree:
+ TableScan
+ alias: lineitem
+ Statistics: Num rows: 100 Data size: 11999 Basic stats:
COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: ((((l_shipmode = 'AIR') and l_orderkey is not
null) and l_linenumber is not null) and (l_linenumber = 1)) (type: boolean)
+ Statistics: Num rows: 6 Data size: 719 Basic stats:
COMPLETE Column stats: NONE
+ Select Operator
+ expressions: l_orderkey (type: int), 1 (type: int)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 6 Data size: 719 Basic stats:
COMPLETE Column stats: NONE
+ Group By Operator
+ keys: _col0 (type: int), _col1 (type: int)
+ mode: hash
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 6 Data size: 719 Basic stats:
COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: int), _col1 (type: int)
+ sort order: ++
+ Map-reduce partition columns: _col0 (type: int),
_col1 (type: int)
+ Statistics: Num rows: 6 Data size: 719 Basic stats:
COMPLETE Column stats: NONE
+ Reducer 2
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Left Semi Join 0 to 1
+ keys:
+ 0 l_orderkey (type: int), l_linenumber (type: int)
+ 1 _col0 (type: int), _col1 (type: int)
+ outputColumnNames: _col1, _col2
+ Statistics: Num rows: 13 Data size: 1582 Basic stats: COMPLETE
Column stats: NONE
+ Select Operator
+ expressions: _col1 (type: int), _col2 (type: int)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 13 Data size: 1582 Basic stats:
COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 13 Data size: 1582 Basic stats:
COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format:
org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde:
org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: SELECT l_partkey, l_suppkey
+FROM lineitem li
+WHERE li.l_linenumber = 1 AND
+ li.l_orderkey IN (SELECT l_orderkey FROM lineitem WHERE l_shipmode = 'AIR'
AND l_linenumber = li.l_linenumber)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@lineitem
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT l_partkey, l_suppkey
+FROM lineitem li
+WHERE li.l_linenumber = 1 AND
+ li.l_orderkey IN (SELECT l_orderkey FROM lineitem WHERE l_shipmode = 'AIR'
AND l_linenumber = li.l_linenumber)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@lineitem
+#### A masked pattern was here ####
+108570 8571
+4297 1798
Added:
hive/branches/spark/ql/src/test/results/clientpositive/spark/empty_dir_in_table.q.out
URL:
http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/empty_dir_in_table.q.out?rev=1656573&view=auto
==============================================================================
---
hive/branches/spark/ql/src/test/results/clientpositive/spark/empty_dir_in_table.q.out
(added)
+++
hive/branches/spark/ql/src/test/results/clientpositive/spark/empty_dir_in_table.q.out
Mon Feb 2 21:10:08 2015
@@ -0,0 +1,46 @@
+#### A masked pattern was here ####
+PREHOOK: type: CREATETABLE
+#### A masked pattern was here ####
+PREHOOK: Output: database:default
+PREHOOK: Output: default@roottable
+#### A masked pattern was here ####
+POSTHOOK: type: CREATETABLE
+#### A masked pattern was here ####
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@roottable
+PREHOOK: query: select count(*) from roottable
+PREHOOK: type: QUERY
+PREHOOK: Input: default@roottable
+#### A masked pattern was here ####
+POSTHOOK: query: select count(*) from roottable
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@roottable
+#### A masked pattern was here ####
+0
+PREHOOK: query: insert into table roottable select key from src where (key <
20) order by key
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@roottable
+POSTHOOK: query: insert into table roottable select key from src where (key <
20) order by key
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@roottable
+POSTHOOK: Lineage: roottable.key SIMPLE [(src)src.FieldSchema(name:key,
type:string, comment:default), ]
+PREHOOK: query: select count(*) from roottable
+PREHOOK: type: QUERY
+PREHOOK: Input: default@roottable
+#### A masked pattern was here ####
+POSTHOOK: query: select count(*) from roottable
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@roottable
+#### A masked pattern was here ####
+20
+PREHOOK: query: select count(*) from roottable
+PREHOOK: type: QUERY
+PREHOOK: Input: default@roottable
+#### A masked pattern was here ####
+POSTHOOK: query: select count(*) from roottable
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@roottable
+#### A masked pattern was here ####
+20
Added:
hive/branches/spark/ql/src/test/results/clientpositive/spark/external_table_with_space_in_location_path.q.out
URL:
http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/external_table_with_space_in_location_path.q.out?rev=1656573&view=auto
==============================================================================
---
hive/branches/spark/ql/src/test/results/clientpositive/spark/external_table_with_space_in_location_path.q.out
(added)
+++
hive/branches/spark/ql/src/test/results/clientpositive/spark/external_table_with_space_in_location_path.q.out
Mon Feb 2 21:10:08 2015
@@ -0,0 +1,88 @@
+#### A masked pattern was here ####
+PREHOOK: type: CREATETABLE
+#### A masked pattern was here ####
+PREHOOK: Output: database:default
+PREHOOK: Output: default@spacetest
+#### A masked pattern was here ####
+POSTHOOK: type: CREATETABLE
+#### A masked pattern was here ####
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@spacetest
+PREHOOK: query: SELECT * FROM spacetest
+PREHOOK: type: QUERY
+PREHOOK: Input: default@spacetest
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT * FROM spacetest
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@spacetest
+#### A masked pattern was here ####
+12 jason
+13 steven
+15 joe
+PREHOOK: query: SELECT count(*) FROM spacetest
+PREHOOK: type: QUERY
+PREHOOK: Input: default@spacetest
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT count(*) FROM spacetest
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@spacetest
+#### A masked pattern was here ####
+3
+PREHOOK: query: DROP TABLE spacetest
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@spacetest
+PREHOOK: Output: default@spacetest
+POSTHOOK: query: DROP TABLE spacetest
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@spacetest
+POSTHOOK: Output: default@spacetest
+PREHOOK: query: CREATE EXTERNAL TABLE spacetestpartition (id int, message
string) PARTITIONED BY (day int) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t'
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@spacetestpartition
+POSTHOOK: query: CREATE EXTERNAL TABLE spacetestpartition (id int, message
string) PARTITIONED BY (day int) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t'
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@spacetestpartition
+#### A masked pattern was here ####
+PREHOOK: type: ALTERTABLE_ADDPARTS
+#### A masked pattern was here ####
+PREHOOK: Output: default@spacetestpartition
+#### A masked pattern was here ####
+POSTHOOK: type: ALTERTABLE_ADDPARTS
+#### A masked pattern was here ####
+POSTHOOK: Output: default@spacetestpartition
+POSTHOOK: Output: default@spacetestpartition@day=10
+PREHOOK: query: SELECT * FROM spacetestpartition
+PREHOOK: type: QUERY
+PREHOOK: Input: default@spacetestpartition
+PREHOOK: Input: default@spacetestpartition@day=10
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT * FROM spacetestpartition
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@spacetestpartition
+POSTHOOK: Input: default@spacetestpartition@day=10
+#### A masked pattern was here ####
+12 jason 10
+13 steven 10
+15 joe 10
+PREHOOK: query: SELECT count(*) FROM spacetestpartition
+PREHOOK: type: QUERY
+PREHOOK: Input: default@spacetestpartition
+PREHOOK: Input: default@spacetestpartition@day=10
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT count(*) FROM spacetestpartition
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@spacetestpartition
+POSTHOOK: Input: default@spacetestpartition@day=10
+#### A masked pattern was here ####
+3
+PREHOOK: query: DROP TABLE spacetestpartition
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@spacetestpartition
+PREHOOK: Output: default@spacetestpartition
+POSTHOOK: query: DROP TABLE spacetestpartition
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@spacetestpartition
+POSTHOOK: Output: default@spacetestpartition
+#### A masked pattern was here ####
Added:
hive/branches/spark/ql/src/test/results/clientpositive/spark/file_with_header_footer.q.out
URL:
http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/file_with_header_footer.q.out?rev=1656573&view=auto
==============================================================================
---
hive/branches/spark/ql/src/test/results/clientpositive/spark/file_with_header_footer.q.out
(added)
+++
hive/branches/spark/ql/src/test/results/clientpositive/spark/file_with_header_footer.q.out
Mon Feb 2 21:10:08 2015
@@ -0,0 +1,184 @@
+#### A masked pattern was here ####
+PREHOOK: type: CREATETABLE
+#### A masked pattern was here ####
+PREHOOK: Output: database:default
+PREHOOK: Output: default@header_footer_table_1
+#### A masked pattern was here ####
+POSTHOOK: type: CREATETABLE
+#### A masked pattern was here ####
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@header_footer_table_1
+PREHOOK: query: SELECT * FROM header_footer_table_1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@header_footer_table_1
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT * FROM header_footer_table_1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@header_footer_table_1
+#### A masked pattern was here ####
+steven hive 1
+dave oozie 2
+xifa phd 3
+chuan hadoop 4
+shanyu senior 5
+steven2 hive 11
+dave2 oozie 12
+xifa2 phd 13
+chuan2 hadoop 14
+shanyu2 senior 15
+david3 oozie 22
+PREHOOK: query: SELECT * FROM header_footer_table_1 WHERE id < 50
+PREHOOK: type: QUERY
+PREHOOK: Input: default@header_footer_table_1
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT * FROM header_footer_table_1 WHERE id < 50
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@header_footer_table_1
+#### A masked pattern was here ####
+steven hive 1
+dave oozie 2
+xifa phd 3
+chuan hadoop 4
+shanyu senior 5
+steven2 hive 11
+dave2 oozie 12
+xifa2 phd 13
+chuan2 hadoop 14
+shanyu2 senior 15
+david3 oozie 22
+PREHOOK: query: CREATE EXTERNAL TABLE header_footer_table_2 (name string,
message string, id int) PARTITIONED BY (year int, month int, day int) ROW
FORMAT DELIMITED FIELDS TERMINATED BY '\t' tblproperties
("skip.header.line.count"="1", "skip.footer.line.count"="2")
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@header_footer_table_2
+POSTHOOK: query: CREATE EXTERNAL TABLE header_footer_table_2 (name string,
message string, id int) PARTITIONED BY (year int, month int, day int) ROW
FORMAT DELIMITED FIELDS TERMINATED BY '\t' tblproperties
("skip.header.line.count"="1", "skip.footer.line.count"="2")
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@header_footer_table_2
+#### A masked pattern was here ####
+PREHOOK: type: ALTERTABLE_ADDPARTS
+#### A masked pattern was here ####
+PREHOOK: Output: default@header_footer_table_2
+#### A masked pattern was here ####
+POSTHOOK: type: ALTERTABLE_ADDPARTS
+#### A masked pattern was here ####
+POSTHOOK: Output: default@header_footer_table_2
+POSTHOOK: Output: default@header_footer_table_2@year=2012/month=1/day=1
+#### A masked pattern was here ####
+PREHOOK: type: ALTERTABLE_ADDPARTS
+#### A masked pattern was here ####
+PREHOOK: Output: default@header_footer_table_2
+#### A masked pattern was here ####
+POSTHOOK: type: ALTERTABLE_ADDPARTS
+#### A masked pattern was here ####
+POSTHOOK: Output: default@header_footer_table_2
+POSTHOOK: Output: default@header_footer_table_2@year=2012/month=1/day=2
+#### A masked pattern was here ####
+PREHOOK: type: ALTERTABLE_ADDPARTS
+#### A masked pattern was here ####
+PREHOOK: Output: default@header_footer_table_2
+#### A masked pattern was here ####
+POSTHOOK: type: ALTERTABLE_ADDPARTS
+#### A masked pattern was here ####
+POSTHOOK: Output: default@header_footer_table_2
+POSTHOOK: Output: default@header_footer_table_2@year=2012/month=1/day=3
+PREHOOK: query: SELECT * FROM header_footer_table_2
+PREHOOK: type: QUERY
+PREHOOK: Input: default@header_footer_table_2
+PREHOOK: Input: default@header_footer_table_2@year=2012/month=1/day=1
+PREHOOK: Input: default@header_footer_table_2@year=2012/month=1/day=2
+PREHOOK: Input: default@header_footer_table_2@year=2012/month=1/day=3
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT * FROM header_footer_table_2
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@header_footer_table_2
+POSTHOOK: Input: default@header_footer_table_2@year=2012/month=1/day=1
+POSTHOOK: Input: default@header_footer_table_2@year=2012/month=1/day=2
+POSTHOOK: Input: default@header_footer_table_2@year=2012/month=1/day=3
+#### A masked pattern was here ####
+steven hive 1 2012 1 1
+dave oozie 2 2012 1 1
+xifa phd 3 2012 1 1
+chuan hadoop 4 2012 1 1
+shanyu senior 5 2012 1 1
+steven2 hive 11 2012 1 2
+dave2 oozie 12 2012 1 2
+xifa2 phd 13 2012 1 2
+chuan2 hadoop 14 2012 1 2
+shanyu2 senior 15 2012 1 2
+david3 oozie 22 2012 1 3
+PREHOOK: query: SELECT * FROM header_footer_table_2 WHERE id < 50
+PREHOOK: type: QUERY
+PREHOOK: Input: default@header_footer_table_2
+PREHOOK: Input: default@header_footer_table_2@year=2012/month=1/day=1
+PREHOOK: Input: default@header_footer_table_2@year=2012/month=1/day=2
+PREHOOK: Input: default@header_footer_table_2@year=2012/month=1/day=3
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT * FROM header_footer_table_2 WHERE id < 50
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@header_footer_table_2
+POSTHOOK: Input: default@header_footer_table_2@year=2012/month=1/day=1
+POSTHOOK: Input: default@header_footer_table_2@year=2012/month=1/day=2
+POSTHOOK: Input: default@header_footer_table_2@year=2012/month=1/day=3
+#### A masked pattern was here ####
+steven hive 1 2012 1 1
+dave oozie 2 2012 1 1
+xifa phd 3 2012 1 1
+chuan hadoop 4 2012 1 1
+shanyu senior 5 2012 1 1
+steven2 hive 11 2012 1 2
+dave2 oozie 12 2012 1 2
+xifa2 phd 13 2012 1 2
+chuan2 hadoop 14 2012 1 2
+shanyu2 senior 15 2012 1 2
+david3 oozie 22 2012 1 3
+#### A masked pattern was here ####
+PREHOOK: type: CREATETABLE
+#### A masked pattern was here ####
+PREHOOK: Output: database:default
+PREHOOK: Output: default@emptytable
+#### A masked pattern was here ####
+POSTHOOK: type: CREATETABLE
+#### A masked pattern was here ####
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@emptytable
+PREHOOK: query: SELECT * FROM emptytable
+PREHOOK: type: QUERY
+PREHOOK: Input: default@emptytable
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT * FROM emptytable
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@emptytable
+#### A masked pattern was here ####
+PREHOOK: query: SELECT * FROM emptytable WHERE id < 50
+PREHOOK: type: QUERY
+PREHOOK: Input: default@emptytable
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT * FROM emptytable WHERE id < 50
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@emptytable
+#### A masked pattern was here ####
+PREHOOK: query: DROP TABLE header_footer_table_1
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@header_footer_table_1
+PREHOOK: Output: default@header_footer_table_1
+POSTHOOK: query: DROP TABLE header_footer_table_1
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@header_footer_table_1
+POSTHOOK: Output: default@header_footer_table_1
+PREHOOK: query: DROP TABLE header_footer_table_2
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@header_footer_table_2
+PREHOOK: Output: default@header_footer_table_2
+POSTHOOK: query: DROP TABLE header_footer_table_2
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@header_footer_table_2
+POSTHOOK: Output: default@header_footer_table_2
+PREHOOK: query: DROP TABLE emptytable
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@emptytable
+PREHOOK: Output: default@emptytable
+POSTHOOK: query: DROP TABLE emptytable
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@emptytable
+POSTHOOK: Output: default@emptytable
+#### A masked pattern was here ####
Added:
hive/branches/spark/ql/src/test/results/clientpositive/spark/import_exported_table.q.out
URL:
http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/import_exported_table.q.out?rev=1656573&view=auto
==============================================================================
---
hive/branches/spark/ql/src/test/results/clientpositive/spark/import_exported_table.q.out
(added)
+++
hive/branches/spark/ql/src/test/results/clientpositive/spark/import_exported_table.q.out
Mon Feb 2 21:10:08 2015
@@ -0,0 +1,25 @@
+#### A masked pattern was here ####
+PREHOOK: type: IMPORT
+#### A masked pattern was here ####
+POSTHOOK: type: IMPORT
+#### A masked pattern was here ####
+POSTHOOK: Output: default@j1_41
+PREHOOK: query: DESCRIBE j1_41
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@j1_41
+POSTHOOK: query: DESCRIBE j1_41
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@j1_41
+a string
+b int
+PREHOOK: query: SELECT * from j1_41
+PREHOOK: type: QUERY
+PREHOOK: Input: default@j1_41
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT * from j1_41
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@j1_41
+#### A masked pattern was here ####
+johndee 1
+burks 2
+#### A masked pattern was here ####
