HIVE-14373: Add integration tests for hive on S3 (Thomas Poepping and Abdullah Yousufi, reviewed by Sergio Pena and Illya Yalovyy)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/527f21b1 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/527f21b1 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/527f21b1 Branch: refs/heads/hive-14535 Commit: 527f21b1750f54b69519e63088755463550e301c Parents: f923db0 Author: Sergio Pena <sergio.p...@cloudera.com> Authored: Thu Oct 13 10:23:19 2016 -0500 Committer: Sergio Pena <sergio.p...@cloudera.com> Committed: Thu Oct 13 10:25:03 2016 -0500 ---------------------------------------------------------------------- .gitignore | 1 + itests/hive-blobstore/README | 25 ++ itests/hive-blobstore/pom.xml | 355 +++++++++++++++++++ .../hadoop/hive/cli/TestBlobstoreCliDriver.java | 64 ++++ .../cli/TestBlobstoreNegativeCliDriver.java | 64 ++++ .../clientnegative/select_dropped_table.q | 4 + .../test/queries/clientpositive/insert_into.q | 4 + .../test/resources/blobstore-conf.xml.template | 22 ++ .../src/test/resources/hive-site.xml | 271 ++++++++++++++ .../test/resources/testconfiguration.properties | 2 + .../src/test/resources/tez-site.xml | 6 + .../clientnegative/select_dropped_table.q.out | 21 ++ .../results/clientpositive/insert_into.q.out | 35 ++ itests/pom.xml | 1 + .../control/AbstractCoreBlobstoreCliDriver.java | 167 +++++++++ .../hadoop/hive/cli/control/CliConfigs.java | 40 +++ .../cli/control/CoreBlobstoreCliDriver.java | 29 ++ .../control/CoreBlobstoreNegativeCliDriver.java | 29 ++ .../org/apache/hadoop/hive/ql/QTestUtil.java | 18 +- 19 files changed, 1155 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/527f21b1/.gitignore ---------------------------------------------------------------------- diff --git a/.gitignore b/.gitignore index 4d341a0..47c59da 100644 --- a/.gitignore +++ b/.gitignore @@ -27,4 +27,5 @@ hcatalog/webhcat/java-client/target hcatalog/storage-handlers/hbase/target hcatalog/webhcat/svr/target conf/hive-default.xml.template +itests/hive-blobstore/src/test/resources/blobstore-conf.xml .DS_Store http://git-wip-us.apache.org/repos/asf/hive/blob/527f21b1/itests/hive-blobstore/README ---------------------------------------------------------------------- diff --git a/itests/hive-blobstore/README b/itests/hive-blobstore/README new file mode 100644 index 0000000..dbd42d5 --- /dev/null +++ b/itests/hive-blobstore/README @@ -0,0 +1,25 @@ +The folder structure details are: + + * ./src/test/queries - contains the queries to be tested on s3 + * ./src/test/results - contains the expected hive console output for the queries + * ./target/qfile-results - Hive console output goes here + * ../../data/conf/blobstore/ - contains hive-site.xml + +To run blobstore integration tests: + + 1. Create blobstore-conf.xml in ./src/test/resources/ with the blobstore credentials and test blobstore path (see blobstore-conf.xml.template). + + 2. Run following command: + mvn test -Dtest=TestBlobstore[Negative]CliDriver + +To run a single integration test: + + 2. Example command: + mvn test -Dtest=TestBlobstoreCliDriver -Dqfile=insert_into.q + +To run the tests on Tez: + + 2. Example command: + mvn test -Dtest=TestBlobstoreCliDriver -Dqfile=insert_into.q -Dhive.execution.engine=tez -Dclustermode=tez_local + +Use ${hiveconf:test.blobstore.path.unique} in .q test files to access the blobstore path. http://git-wip-us.apache.org/repos/asf/hive/blob/527f21b1/itests/hive-blobstore/pom.xml ---------------------------------------------------------------------- diff --git a/itests/hive-blobstore/pom.xml b/itests/hive-blobstore/pom.xml new file mode 100644 index 0000000..a62885f --- /dev/null +++ b/itests/hive-blobstore/pom.xml @@ -0,0 +1,355 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<project xmlns="http://maven.apache.org/POM/4.0.0"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + + <parent> + <groupId>org.apache.hive</groupId> + <artifactId>hive-it</artifactId> + <version>2.2.0-SNAPSHOT</version> + <relativePath>../pom.xml</relativePath> + </parent> + + <artifactId>hive-blobstore</artifactId> + <packaging>jar</packaging> + <name>Hive Integration - Blobstore Tests</name> + + <properties> + <hive.path.to.root>../..</hive.path.to.root> + <qfile></qfile> + <qfile_regex></qfile_regex> + <run_disabled>false</run_disabled> + <clustermode></clustermode> + <execute.beeline.tests>false</execute.beeline.tests> + <active.hadoop.version>${hadoop.version}</active.hadoop.version> + <test.dfs.mkdir>-mkdir -p</test.dfs.mkdir> + </properties> + + <profiles> + <profile> + <id>tests-off</id> + <activation> + <file> + <missing>src/test/resources/blobstore-conf.xml</missing> + </file> + </activation> + <properties> + <maven.test.skip>true</maven.test.skip> + </properties> + </profile> + <profile> + <id>tests-on</id> + <activation> + <file> + <exists>src/test/resources/blobstore-conf.xml</exists> + </file> + </activation> + <properties> + <maven.test.skip>false</maven.test.skip> + </properties> + </profile> + </profiles> + + <dependencies> + <!-- dependencies are always listed in sorted order by groupId, artifactId --> + <dependency> + <groupId>org.apache.hive</groupId> + <artifactId>hive-ant</artifactId> + <version>${project.version}</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.apache.hive</groupId> + <artifactId>hive-common</artifactId> + <version>${project.version}</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.apache.hive</groupId> + <artifactId>hive-contrib</artifactId> + <version>${project.version}</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.apache.hive</groupId> + <artifactId>hive-metastore</artifactId> + <version>${project.version}</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.apache.hive</groupId> + <artifactId>hive-metastore</artifactId> + <version>${project.version}</version> + <classifier>tests</classifier> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.apache.hive</groupId> + <artifactId>hive-it-unit</artifactId> + <version>${project.version}</version> + <classifier>tests</classifier> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.apache.hive</groupId> + <artifactId>hive-serde</artifactId> + <version>${project.version}</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.apache.hive</groupId> + <artifactId>hive-exec</artifactId> + <version>${project.version}</version> + <scope>test</scope> + </dependency> + + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-common</artifactId> + <version>${hadoop.version}</version> + <scope>test</scope> + <exclusions> + <exclusion> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-log4j12</artifactId> + </exclusion> + <exclusion> + <groupId>commmons-logging</groupId> + <artifactId>commons-logging</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-common</artifactId> + <version>${hadoop.version}</version> + <classifier>tests</classifier> + <scope>test</scope> + <exclusions> + <exclusion> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-log4j12</artifactId> + </exclusion> + <exclusion> + <groupId>commmons-logging</groupId> + <artifactId>commons-logging</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-mapreduce-client-jobclient</artifactId> + <version>${hadoop.version}</version> + <classifier>tests</classifier> + <scope>test</scope> + <exclusions> + <exclusion> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-log4j12</artifactId> + </exclusion> + <exclusion> + <groupId>commmons-logging</groupId> + <artifactId>commons-logging</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-mapreduce-client-hs</artifactId> + <version>${hadoop.version}</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-mapreduce-client-core</artifactId> + <version>${hadoop.version}</version> + <scope>test</scope> + <exclusions> + <exclusion> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-log4j12</artifactId> + </exclusion> + <exclusion> + <groupId>commmons-logging</groupId> + <artifactId>commons-logging</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>org.apache.tez</groupId> + <artifactId>tez-tests</artifactId> + <version>${tez.version}</version> + <type>test-jar</type> + </dependency> + <dependency> + <groupId>org.apache.tez</groupId> + <artifactId>tez-api</artifactId> + <version>${tez.version}</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.apache.tez</groupId> + <artifactId>tez-runtime-library</artifactId> + <version>${tez.version}</version> + <scope>test</scope> + <exclusions> + <exclusion> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-log4j12</artifactId> + </exclusion> + <exclusion> + <groupId>commmons-logging</groupId> + <artifactId>commons-logging</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>org.apache.tez</groupId> + <artifactId>tez-mapreduce</artifactId> + <version>${tez.version}</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.apache.tez</groupId> + <artifactId>tez-dag</artifactId> + <version>${tez.version}</version> + <scope>test</scope> + <exclusions> + <exclusion> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-log4j12</artifactId> + </exclusion> + <exclusion> + <groupId>commmons-logging</groupId> + <artifactId>commons-logging</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <version>${junit.version}</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-aws</artifactId> + <version>${hadoop.version}</version> + <scope>compile</scope> + <exclusions> + <exclusion> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-log4j12</artifactId> + </exclusion> + <exclusion> + <groupId>commmons-logging</groupId> + <artifactId>commons-logging</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>com.fasterxml.jackson.core</groupId> + <artifactId>jackson-annotations</artifactId> + <version>${jackson.new.version}</version> + </dependency> + <dependency> + <groupId>com.fasterxml.jackson.core</groupId> + <artifactId>jackson-core</artifactId> + <version>${jackson.new.version}</version> + </dependency> + <dependency> + <groupId>com.fasterxml.jackson.core</groupId> + <artifactId>jackson-databind</artifactId> + <version>${jackson.new.version}</version> + </dependency> + </dependencies> + + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-surefire-plugin</artifactId> + <configuration> + <systemPropertyVariables> + <test.blobstore.path>${test.blobstore.path}</test.blobstore.path> + </systemPropertyVariables> + </configuration> + </plugin> + <plugin> + <groupId>org.codehaus.mojo</groupId> + <artifactId>properties-maven-plugin</artifactId> + <version>1.0-alpha-2</version> + <executions> + <execution> + <phase>initialize</phase> + <goals> + <goal>read-project-properties</goal> + </goals> + <configuration> + <files> + <file>${basedir}/../src/test/resources/testconfiguration.properties</file> + </files> + </configuration> + </execution> + </executions> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-antrun-plugin</artifactId> + <executions> + <execution> + <id>generate-tests-sources</id> + <phase>generate-test-sources</phase> + <configuration> + <target> + <property name="test.classpath" refid="maven.test.classpath"/> + <echo message="${test.classpath}"/> + <taskdef resource="net/sf/antcontrib/antcontrib.properties" + classpathref="maven.plugin.classpath" /> + <mkdir dir="${project.build.directory}/qfile-results/clientpositive" /> + <mkdir dir="${project.build.directory}/qfile-results/clientnegative" /> + </target> + </configuration> + <goals> + <goal>run</goal> + </goals> + </execution> + </executions> + </plugin> + <plugin> + <groupId>org.codehaus.mojo</groupId> + <artifactId>build-helper-maven-plugin</artifactId> + <version>${maven.build-helper.plugin.version}</version> + <executions> + <execution> + <id>add-test-sources</id> + <phase>generate-test-sources</phase> + <goals> + <goal>add-test-source</goal> + </goals> + <configuration> + <sources> + <source>target/generated-test-sources/java</source> + </sources> + </configuration> + </execution> + </executions> + </plugin> + </plugins> + </build> + +</project> http://git-wip-us.apache.org/repos/asf/hive/blob/527f21b1/itests/hive-blobstore/src/test/java/org/apache/hadoop/hive/cli/TestBlobstoreCliDriver.java ---------------------------------------------------------------------- diff --git a/itests/hive-blobstore/src/test/java/org/apache/hadoop/hive/cli/TestBlobstoreCliDriver.java b/itests/hive-blobstore/src/test/java/org/apache/hadoop/hive/cli/TestBlobstoreCliDriver.java new file mode 100644 index 0000000..4aee385 --- /dev/null +++ b/itests/hive-blobstore/src/test/java/org/apache/hadoop/hive/cli/TestBlobstoreCliDriver.java @@ -0,0 +1,64 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hadoop.hive.cli; + +import java.io.File; +import java.util.List; + +import org.apache.hadoop.hive.cli.control.CliAdapter; +import org.apache.hadoop.hive.cli.control.CliConfigs; +import org.junit.ClassRule; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TestRule; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameters; + +@RunWith(Parameterized.class) +public class TestBlobstoreCliDriver { + + static CliAdapter adapter = new CliConfigs.BlobstoreCliConfig().getCliAdapter(); + + @Parameters(name = "{0}") + public static List<Object[]> getParameters() throws Exception { + return adapter.getParameters(); + } + + @ClassRule + public static TestRule cliClassRule = adapter.buildClassRule(); + + @Rule + public TestRule cliTestRule = adapter.buildTestRule(); + + private String name; + private File qfile; + + public TestBlobstoreCliDriver(String name, File qfile) { + this.name = name; + this.qfile = qfile; + } + + @Test + public void testCliDriver() throws Exception { + adapter.runTest(name, qfile); + } + +} http://git-wip-us.apache.org/repos/asf/hive/blob/527f21b1/itests/hive-blobstore/src/test/java/org/apache/hadoop/hive/cli/TestBlobstoreNegativeCliDriver.java ---------------------------------------------------------------------- diff --git a/itests/hive-blobstore/src/test/java/org/apache/hadoop/hive/cli/TestBlobstoreNegativeCliDriver.java b/itests/hive-blobstore/src/test/java/org/apache/hadoop/hive/cli/TestBlobstoreNegativeCliDriver.java new file mode 100644 index 0000000..90f1b47 --- /dev/null +++ b/itests/hive-blobstore/src/test/java/org/apache/hadoop/hive/cli/TestBlobstoreNegativeCliDriver.java @@ -0,0 +1,64 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hadoop.hive.cli; + +import java.io.File; +import java.util.List; + +import org.apache.hadoop.hive.cli.control.CliAdapter; +import org.apache.hadoop.hive.cli.control.CliConfigs; +import org.junit.ClassRule; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TestRule; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameters; + +@RunWith(Parameterized.class) +public class TestBlobstoreNegativeCliDriver { + + static CliAdapter adapter = new CliConfigs.BlobstoreNegativeCliConfig().getCliAdapter(); + + @Parameters(name = "{0}") + public static List<Object[]> getParameters() throws Exception { + return adapter.getParameters(); + } + + @ClassRule + public static TestRule cliClassRule = adapter.buildClassRule(); + + @Rule + public TestRule cliTestRule = adapter.buildTestRule(); + + private String name; + private File qfile; + + public TestBlobstoreNegativeCliDriver(String name, File qfile) { + this.name = name; + this.qfile = qfile; + } + + @Test + public void testCliDriver() throws Exception { + adapter.runTest(name, qfile); + } + +} http://git-wip-us.apache.org/repos/asf/hive/blob/527f21b1/itests/hive-blobstore/src/test/queries/clientnegative/select_dropped_table.q ---------------------------------------------------------------------- diff --git a/itests/hive-blobstore/src/test/queries/clientnegative/select_dropped_table.q b/itests/hive-blobstore/src/test/queries/clientnegative/select_dropped_table.q new file mode 100644 index 0000000..7adc841 --- /dev/null +++ b/itests/hive-blobstore/src/test/queries/clientnegative/select_dropped_table.q @@ -0,0 +1,4 @@ +CREATE TABLE qtest (key STRING, value STRING) +LOCATION '${hiveconf:test.blobstore.path.unique}/qtest'; +DROP TABLE qtest; +SELECT * FROM qtest; http://git-wip-us.apache.org/repos/asf/hive/blob/527f21b1/itests/hive-blobstore/src/test/queries/clientpositive/insert_into.q ---------------------------------------------------------------------- diff --git a/itests/hive-blobstore/src/test/queries/clientpositive/insert_into.q b/itests/hive-blobstore/src/test/queries/clientpositive/insert_into.q new file mode 100644 index 0000000..e36ef1d --- /dev/null +++ b/itests/hive-blobstore/src/test/queries/clientpositive/insert_into.q @@ -0,0 +1,4 @@ +DROP TABLE qtest; +CREATE TABLE qtest (value int) LOCATION '${hiveconf:test.blobstore.path.unique}/qtest/'; +INSERT INTO qtest VALUES (1), (10), (100), (1000); +SELECT * FROM qtest; http://git-wip-us.apache.org/repos/asf/hive/blob/527f21b1/itests/hive-blobstore/src/test/resources/blobstore-conf.xml.template ---------------------------------------------------------------------- diff --git a/itests/hive-blobstore/src/test/resources/blobstore-conf.xml.template b/itests/hive-blobstore/src/test/resources/blobstore-conf.xml.template new file mode 100644 index 0000000..f53cfa6 --- /dev/null +++ b/itests/hive-blobstore/src/test/resources/blobstore-conf.xml.template @@ -0,0 +1,22 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +# Copy this file to blobstore-conf.xml +# update the properties + credentials according to the blobstore being tested + +# The following is an example template file for s3a +--> +<configuration> + <property> + <name>fs.s3a.access.key</name> + <value></value> + </property> + <property> + <name>fs.s3a.secret.key</name> + <value></value> + </property> + <property> + <name>test.blobstore.path</name> + <value></value> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/hive/blob/527f21b1/itests/hive-blobstore/src/test/resources/hive-site.xml ---------------------------------------------------------------------- diff --git a/itests/hive-blobstore/src/test/resources/hive-site.xml b/itests/hive-blobstore/src/test/resources/hive-site.xml new file mode 100644 index 0000000..ffac40a --- /dev/null +++ b/itests/hive-blobstore/src/test/resources/hive-site.xml @@ -0,0 +1,271 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<configuration> + <property> + <name>hive.in.test</name> + <value>true</value> + <description>Internal marker for test. Used for masking env-dependent values</description> + </property> + + <!-- Hive Configuration can either be stored in this file or in the hadoop configuration files --> + <!-- that are implied by Hadoop setup variables. --> + <!-- Aside from Hadoop setup variables - this file is provided as a convenience so that Hive --> + <!-- users do not have to edit hadoop configuration files (that may be managed as a centralized --> + <!-- resource). --> + + <!-- Hive Execution Parameters --> + <property> + <name>hadoop.tmp.dir</name> + <value>${test.tmp.dir}/hadoop-tmp</value> + <description>A base for other temporary directories.</description> + </property> + + <!-- + <property> + <name>hive.exec.reducers.max</name> + <value>1</value> + <description>maximum number of reducers</description> + </property> + --> + + <property> + <name>hive.exec.scratchdir</name> + <value>${test.tmp.dir}/scratchdir</value> + <description>Scratch space for Hive jobs</description> + </property> + + <property> + <name>hive.exec.local.scratchdir</name> + <value>${test.tmp.dir}/localscratchdir/</value> + <description>Local scratch space for Hive jobs</description> + </property> + + <property> + <name>local.tmp.dir</name> + <value>file:///${env:PWD}/target/tmp</value> + </property> + + <property> + <name>datanucleus.schema.autoCreateAll</name> + <value>true</value> + </property> + + <property> + <name>hive.metastore.schema.verification</name> + <value>false</value> + </property> + + <property> + <name>javax.jdo.option.ConnectionURL</name> + <value>jdbc:derby:;databaseName=${test.tmp.dir}/junit_metastore_db;create=true</value> + </property> + + <property> + <name>hive.stats.dbconnectionstring</name> + <value>jdbc:derby:;databaseName=${test.tmp.dir}/TempStatsStore;create=true</value> + </property> + + <property> + <name>javax.jdo.option.ConnectionDriverName</name> + <value>org.apache.derby.jdbc.EmbeddedDriver</value> + </property> + + <property> + <name>javax.jdo.option.ConnectionUserName</name> + <value>APP</value> + </property> + + <property> + <name>javax.jdo.option.ConnectionPassword</name> + <value>mine</value> + </property> + + <property> + <!-- this should eventually be deprecated since the metastore should supply this --> + <name>hive.metastore.warehouse.dir</name> + <value>${test.warehouse.dir}</value> + <description></description> + </property> + + <property> + <name>hive.metastore.metadb.dir</name> + <value>file://${test.tmp.dir}/metadb/</value> + <description> + Required by metastore server or if the uris argument below is not supplied + </description> + </property> + + <property> + <name>test.log.dir</name> + <value>${test.tmp.dir}/log/</value> + <description></description> + </property> + + <property> + <name>test.data.files</name> + <value>${hive.root}/data/files</value> + <description></description> + </property> + + <property> + <name>test.data.scripts</name> + <value>${basedir}/src/test/data/scripts</value> + <description></description> + </property> + + <property> + <name>hive.jar.path</name> + <value>${maven.local.repository}/org/apache/hive/hive-exec/${hive.version}/hive-exec-${hive.version}.jar</value> + <description></description> + </property> + + <property> + <name>hive.metastore.rawstore.impl</name> + <value>org.apache.hadoop.hive.metastore.ObjectStore</value> + <description>Name of the class that implements org.apache.hadoop.hive.metastore.rawstore interface. This class is used to store and retrieval of raw metadata objects such as table, database</description> + </property> + + <property> + <name>hive.querylog.location</name> + <value>${test.tmp.dir}/tmp</value> + <description>Location of the structured hive logs</description> + </property> + + <property> + <name>hive.exec.pre.hooks</name> + <value>org.apache.hadoop.hive.ql.hooks.PreExecutePrinter, org.apache.hadoop.hive.ql.hooks.EnforceReadOnlyTables</value> + <description>Pre Execute Hook for Tests</description> + </property> + + <property> + <name>hive.exec.post.hooks</name> + <value>org.apache.hadoop.hive.ql.hooks.PostExecutePrinter</value> + <description>Post Execute Hook for Tests</description> + </property> + + <property> + <name>hive.support.concurrency</name> + <value>true</value> + <description>Whether hive supports concurrency or not. A zookeeper instance must be up and running for the default hive lock manager to support read-write locks.</description> + </property> + + <property> + <key>hive.unlock.numretries</key> + <value>2</value> + <description>The number of times you want to retry to do one unlock</description> + </property> + + <property> + <key>hive.lock.sleep.between.retries</key> + <value>2</value> + <description>The sleep time (in seconds) between various retries</description> + </property> + + <property> + <name>fs.pfile.impl</name> + <value>org.apache.hadoop.fs.ProxyLocalFileSystem</value> + <description>A proxy for local file system used for cross file system testing</description> + </property> + + <property> + <name>hive.exec.mode.local.auto</name> + <value>false</value> + <description> + Let hive determine whether to run in local mode automatically + Disabling this for tests so that minimr is not affected + </description> + </property> + + <property> + <name>hive.auto.convert.join</name> + <value>false</value> + <description>Whether Hive enable the optimization about converting common join into mapjoin based on the input file size</description> + </property> + + <property> + <name>hive.ignore.mapjoin.hint</name> + <value>false</value> + <description>Whether Hive ignores the mapjoin hint</description> + </property> + + <property> + <name>hive.input.format</name> + <value>org.apache.hadoop.hive.ql.io.CombineHiveInputFormat</value> + <description>The default input format, if it is not specified, the system assigns it. It is set to HiveInputFormat for hadoop versions 17, 18 and 19, whereas it is set to CombineHiveInputFormat for hadoop 20. The user can always overwrite it - if there is a bug in CombineHiveInputFormat, it can always be manually set to HiveInputFormat. </description> + </property> + + <property> + <name>hive.default.rcfile.serde</name> + <value>org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe</value> + <description>The default SerDe hive will use for the rcfile format</description> + </property> + + <property> + <name>hive.stats.key.prefix.reserve.length</name> + <value>0</value> + </property> + + <property> + <name>hive.conf.restricted.list</name> + <value>dummy.config.value</value> + <description>Using dummy config value above because you cannot override config with empty value</description> + </property> + + <property> + <name>hive.exec.submit.local.task.via.child</name> + <value>false</value> + </property> + + <property> + <name>hive.optimize.sort.dynamic.partition</name> + <value>true</value> + </property> + + <property> + <name>hive.dummyparam.test.server.specific.config.override</name> + <value>from.hive-site.xml</value> + <description>Using dummy param to test server specific configuration</description> + </property> + + <property> + <name>hive.dummyparam.test.server.specific.config.hivesite</name> + <value>from.hive-site.xml</value> + <description>Using dummy param to test server specific configuration</description> + </property> + + <property> + <name>hive.fetch.task.conversion</name> + <value>minimal</value> + </property> + + <property> + <name>hive.users.in.admin.role</name> + <value>hive_admin_user</value> + </property> + + <!-- + To run these tests: + # Create a file blobstore-conf.xml - DO NOT ADD TO REVISION CONTROL + # Add the credentials for the service you are testing against + --> + <include xmlns="http://www.w3.org/2001/XInclude"; href="blobstore-conf.xml"> + <fallback/> + </include> +</configuration> http://git-wip-us.apache.org/repos/asf/hive/blob/527f21b1/itests/hive-blobstore/src/test/resources/testconfiguration.properties ---------------------------------------------------------------------- diff --git a/itests/hive-blobstore/src/test/resources/testconfiguration.properties b/itests/hive-blobstore/src/test/resources/testconfiguration.properties new file mode 100644 index 0000000..1db1510 --- /dev/null +++ b/itests/hive-blobstore/src/test/resources/testconfiguration.properties @@ -0,0 +1,2 @@ +include.query.files= +exclude.query.files= http://git-wip-us.apache.org/repos/asf/hive/blob/527f21b1/itests/hive-blobstore/src/test/resources/tez-site.xml ---------------------------------------------------------------------- diff --git a/itests/hive-blobstore/src/test/resources/tez-site.xml b/itests/hive-blobstore/src/test/resources/tez-site.xml new file mode 100644 index 0000000..940f390 --- /dev/null +++ b/itests/hive-blobstore/src/test/resources/tez-site.xml @@ -0,0 +1,6 @@ +<configuration> + <property> + <name>tez.am.dag.scheduler.class</name> + <value>org.apache.tez.dag.app.dag.impl.DAGSchedulerNaturalOrderControlled</value> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/hive/blob/527f21b1/itests/hive-blobstore/src/test/results/clientnegative/select_dropped_table.q.out ---------------------------------------------------------------------- diff --git a/itests/hive-blobstore/src/test/results/clientnegative/select_dropped_table.q.out b/itests/hive-blobstore/src/test/results/clientnegative/select_dropped_table.q.out new file mode 100644 index 0000000..e227b36 --- /dev/null +++ b/itests/hive-blobstore/src/test/results/clientnegative/select_dropped_table.q.out @@ -0,0 +1,21 @@ +PREHOOK: query: CREATE TABLE qtest (key STRING, value STRING) +#### A masked pattern was here #### +PREHOOK: type: CREATETABLE +PREHOOK: Input: #### A masked pattern was here #### +PREHOOK: Output: database:default +PREHOOK: Output: default@qtest +POSTHOOK: query: CREATE TABLE qtest (key STRING, value STRING) +#### A masked pattern was here #### +POSTHOOK: type: CREATETABLE +POSTHOOK: Input: #### A masked pattern was here #### +POSTHOOK: Output: database:default +POSTHOOK: Output: default@qtest +PREHOOK: query: DROP TABLE qtest +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@qtest +PREHOOK: Output: default@qtest +POSTHOOK: query: DROP TABLE qtest +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@qtest +POSTHOOK: Output: default@qtest +FAILED: SemanticException [Error 10001]: Line 2:14 Table not found 'qtest' http://git-wip-us.apache.org/repos/asf/hive/blob/527f21b1/itests/hive-blobstore/src/test/results/clientpositive/insert_into.q.out ---------------------------------------------------------------------- diff --git a/itests/hive-blobstore/src/test/results/clientpositive/insert_into.q.out b/itests/hive-blobstore/src/test/results/clientpositive/insert_into.q.out new file mode 100644 index 0000000..919f3e7 --- /dev/null +++ b/itests/hive-blobstore/src/test/results/clientpositive/insert_into.q.out @@ -0,0 +1,35 @@ +PREHOOK: query: DROP TABLE qtest +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE qtest +POSTHOOK: type: DROPTABLE +#### A masked pattern was here #### +PREHOOK: type: CREATETABLE +PREHOOK: Input: #### A masked pattern was here #### +PREHOOK: Output: database:default +PREHOOK: Output: default@qtest +#### A masked pattern was here #### +POSTHOOK: type: CREATETABLE +POSTHOOK: Input: #### A masked pattern was here #### +POSTHOOK: Output: database:default +POSTHOOK: Output: default@qtest +PREHOOK: query: INSERT INTO qtest VALUES (1), (10), (100), (1000) +PREHOOK: type: QUERY +PREHOOK: Input: default@values__tmp__table__1 +PREHOOK: Output: default@qtest +POSTHOOK: query: INSERT INTO qtest VALUES (1), (10), (100), (1000) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@values__tmp__table__1 +POSTHOOK: Output: default@qtest +POSTHOOK: Lineage: qtest.value EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +PREHOOK: query: SELECT * FROM qtest +PREHOOK: type: QUERY +PREHOOK: Input: default@qtest +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM qtest +POSTHOOK: type: QUERY +POSTHOOK: Input: default@qtest +#### A masked pattern was here #### +1 +10 +100 +1000 http://git-wip-us.apache.org/repos/asf/hive/blob/527f21b1/itests/pom.xml ---------------------------------------------------------------------- diff --git a/itests/pom.xml b/itests/pom.xml index a452db3..e5b54bf 100644 --- a/itests/pom.xml +++ b/itests/pom.xml @@ -36,6 +36,7 @@ <module>custom-udfs</module> <module>hcatalog-unit</module> <module>hive-unit</module> + <module>hive-blobstore</module> <module>util</module> <module>test-serde</module> <module>qtest</module> http://git-wip-us.apache.org/repos/asf/hive/blob/527f21b1/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCoreBlobstoreCliDriver.java ---------------------------------------------------------------------- diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCoreBlobstoreCliDriver.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCoreBlobstoreCliDriver.java new file mode 100644 index 0000000..e797eb5 --- /dev/null +++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCoreBlobstoreCliDriver.java @@ -0,0 +1,167 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.cli.control; + +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import com.google.common.base.Strings; +import java.text.SimpleDateFormat; +import java.util.Calendar; +import org.apache.hadoop.hive.cli.control.AbstractCliConfig.MetastoreType; +import org.apache.hadoop.hive.ql.QTestUtil; +import org.apache.hadoop.hive.ql.QTestUtil.MiniClusterType; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; + +public abstract class AbstractCoreBlobstoreCliDriver extends CliAdapter { + + protected static QTestUtil qt; + private static final String HCONF_TEST_BLOBSTORE_PATH = "test.blobstore.path"; + private static final String HCONF_TEST_BLOBSTORE_PATH_UNIQUE = HCONF_TEST_BLOBSTORE_PATH + ".unique"; + private static String testBlobstorePathUnique; + + public AbstractCoreBlobstoreCliDriver(AbstractCliConfig testCliConfig) { + super(testCliConfig); + } + + @Override + @BeforeClass + public void beforeClass() { + MiniClusterType miniMR = cliConfig.getClusterType(); + String hiveConfDir = cliConfig.getHiveConfDir(); + String initScript = cliConfig.getInitScript(); + String cleanupScript = cliConfig.getCleanupScript(); + boolean useHBaseMetastore = cliConfig.getMetastoreType() == MetastoreType.hbase; + try { + String hadoopVer = cliConfig.getHadoopVersion(); + qt = new QTestUtil((cliConfig.getResultsDir()), (cliConfig.getLogDir()), miniMR, + hiveConfDir, hadoopVer, initScript, cleanupScript, useHBaseMetastore, true); + + if (Strings.isNullOrEmpty(qt.getConf().get(HCONF_TEST_BLOBSTORE_PATH))) { + fail(String.format("%s must be set. Try setting in blobstore-conf.xml", HCONF_TEST_BLOBSTORE_PATH)); + } + + // do a one time initialization + setupUniqueTestPath(); + qt.cleanUp(); + qt.createSources(); + } catch (Exception e) { + System.err.println("Exception: " + e.getMessage()); + e.printStackTrace(); + System.err.flush(); + throw new RuntimeException("Unexpected exception in static initialization",e); + } + } + + @Override + @Before + public void setUp() { + try { + qt.clearTestSideEffects(); + } catch (Exception e) { + System.err.println("Exception: " + e.getMessage()); + e.printStackTrace(); + System.err.flush(); + fail("Unexpected exception in setup"); + } + } + + @Override + @After + public void tearDown() { + try { + qt.clearPostTestEffects(); + } catch (Exception e) { + System.err.println("Exception: " + e.getMessage()); + e.printStackTrace(); + System.err.flush(); + fail("Unexpected exception in tearDown"); + } + } + + @Override + @AfterClass + public void shutdown() throws Exception { + try { + qt.shutdown(); + if (System.getenv(QTestUtil.QTEST_LEAVE_FILES) == null) { + String rmUniquePathCommand = String.format("dfs -rmdir ${hiveconf:%s};", HCONF_TEST_BLOBSTORE_PATH_UNIQUE); + qt.executeAdhocCommand(rmUniquePathCommand); + } + } catch (Exception e) { + System.err.println("Exception: " + e.getMessage()); + e.printStackTrace(); + System.err.flush(); + fail("Unexpected exception in shutdown"); + } + } + + static String debugHint = "\nSee ./itests/hive-blobstore/target/tmp/log/hive.log, " + + "or check ./itests/hive-blobstore/target/surefire-reports/ for specific test cases logs."; + + protected void runTestHelper(String tname, String fname, String fpath, boolean expectSuccess) throws Exception { + long startTime = System.currentTimeMillis(); + qt.getConf().set(HCONF_TEST_BLOBSTORE_PATH_UNIQUE, testBlobstorePathUnique); + try { + System.err.println("Begin query: " + fname); + + qt.addFile(fpath); + + if (qt.shouldBeSkipped(fname)) { + System.err.println("Test " + fname + " skipped"); + return; + } + qt.cliInit(fname, false); + int ecode = qt.executeClient(fname); + if ((ecode == 0) ^ expectSuccess) { + qt.failed(ecode, fname, debugHint); + } + ecode = qt.checkCliDriverResults(fname); + if (ecode != 0) { + qt.failedDiff(ecode, fname, debugHint); + } + } + catch (Throwable e) { + qt.failed(e, fname, debugHint); + } + + long elapsedTime = System.currentTimeMillis() - startTime; + System.err.println("Done query: " + fname + " elapsedTime=" + elapsedTime/1000 + "s"); + assertTrue("Test passed", true); + } + + /** + * Generates a unique test path for this particular CliDriver in the following form: + * ${test.blobstore.path}/CoreBlobstore[Negative]CliDriver/20160101.053046.332-{random number 000-999} + * 20160101.053046.332 represents the current datetime: + * {year}{month}{day}.{hour}{minute}{second}.{millisecond} + * Random integer 000-999 included to avoid collisions when two test runs are started at the same millisecond with + * the same ${test.blobstore.path} (possible if test runs are controlled by an automated system) + */ + private void setupUniqueTestPath() { + String testBlobstorePath = qt.getConf().get(HCONF_TEST_BLOBSTORE_PATH); + testBlobstorePath = QTestUtil.ensurePathEndsInSlash(testBlobstorePath); + testBlobstorePath += QTestUtil.ensurePathEndsInSlash(this.getClass().getSimpleName()); // name of child class + String uid = new SimpleDateFormat("yyyyMMdd.HHmmss.SSS").format(Calendar.getInstance().getTime()) + + "-" + String.format("%03d", (int)(Math.random() * 999)); + testBlobstorePathUnique = testBlobstorePath + uid; + } +} http://git-wip-us.apache.org/repos/asf/hive/blob/527f21b1/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java ---------------------------------------------------------------------- diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java index e7748b7..366c7b4 100644 --- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java +++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java @@ -515,4 +515,44 @@ public class CliConfigs { } } } + + public static class BlobstoreCliConfig extends AbstractCliConfig { + public BlobstoreCliConfig() { + super(CoreBlobstoreCliDriver.class); + try { + setQueryDir("itests/hive-blobstore/src/test/queries/clientpositive"); + + setResultsDir("itests/hive-blobstore/src/test/results/clientpositive"); + setLogDir("itests/hive-blobstore/target/qfile-results/clientpositive"); + + setInitScript("blobstore_test_init.q"); + setCleanupScript("blobstore_test_cleanup.q"); + + setHiveConfDir("itests/hive-blobstore/src/test/resources"); + setClusterType(MiniClusterType.none); + } catch (Exception e) { + throw new RuntimeException("can't construct cliconfig", e); + } + } + } + + public static class BlobstoreNegativeCliConfig extends AbstractCliConfig { + public BlobstoreNegativeCliConfig() { + super(CoreBlobstoreNegativeCliDriver.class); + try { + setQueryDir("itests/hive-blobstore/src/test/queries/clientnegative"); + + setResultsDir("itests/hive-blobstore/src/test/results/clientnegative"); + setLogDir("itests/hive-blobstore/target/qfile-results/clientnegative"); + + setInitScript("blobstore_test_init.q"); + setCleanupScript("blobstore_test_cleanup.q"); + + setHiveConfDir("itests/hive-blobstore/src/test/resources"); + setClusterType(MiniClusterType.none); + } catch (Exception e) { + throw new RuntimeException("can't construct cliconfig", e); + } + } + } } http://git-wip-us.apache.org/repos/asf/hive/blob/527f21b1/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreBlobstoreCliDriver.java ---------------------------------------------------------------------- diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreBlobstoreCliDriver.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreBlobstoreCliDriver.java new file mode 100644 index 0000000..42b2276 --- /dev/null +++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreBlobstoreCliDriver.java @@ -0,0 +1,29 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.cli.control; + +public class CoreBlobstoreCliDriver extends AbstractCoreBlobstoreCliDriver { + public CoreBlobstoreCliDriver(AbstractCliConfig testCliConfig) { + super(testCliConfig); + } + + @Override + public void runTest(String tname, String fname, String fpath) throws Exception { + super.runTestHelper(tname, fname, fpath, true); + } +} http://git-wip-us.apache.org/repos/asf/hive/blob/527f21b1/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreBlobstoreNegativeCliDriver.java ---------------------------------------------------------------------- diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreBlobstoreNegativeCliDriver.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreBlobstoreNegativeCliDriver.java new file mode 100644 index 0000000..b63a71f --- /dev/null +++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreBlobstoreNegativeCliDriver.java @@ -0,0 +1,29 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.cli.control; + +public class CoreBlobstoreNegativeCliDriver extends AbstractCoreBlobstoreCliDriver { + public CoreBlobstoreNegativeCliDriver(AbstractCliConfig testCliConfig) { + super(testCliConfig); + } + + @Override + public void runTest(String tname, String fname, String fpath) throws Exception { + super.runTestHelper(tname, fname, fpath, false); + } +} http://git-wip-us.apache.org/repos/asf/hive/blob/527f21b1/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java ---------------------------------------------------------------------- diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java index 5f683f0..ff640d3 100644 --- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java +++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java @@ -142,8 +142,8 @@ public class QTestUtil { private static final String SECURITY_KEY_PROVIDER_URI_NAME = "dfs.encryption.key.provider.uri"; private static final String CRLF = System.getProperty("line.separator"); + public static final String QTEST_LEAVE_FILES = "QTEST_LEAVE_FILES"; private static final Logger LOG = LoggerFactory.getLogger("QTestUtil"); - private static final String QTEST_LEAVE_FILES = "QTEST_LEAVE_FILES"; private final static String defaultInitScript = "q_test_init.sql"; private final static String defaultCleanupScript = "q_test_cleanup.sql"; private final String[] testOnlyCommands = new String[]{"crypto"}; @@ -1259,6 +1259,17 @@ public class QTestUtil { return ss; } + public int executeAdhocCommand(String q) { + if (!q.contains(";")) { + return -1; + } + + String q1 = q.split(";")[0] + ";"; + + LOG.debug("Executing " + q1); + return cliDriver.processLine(q1); + } + public int executeOne(String tname) { String q = qMap.get(tname); @@ -1655,7 +1666,8 @@ public class QTestUtil { "fk_-?[0-9]*_[0-9]*_[0-9]*", ".*at com\\.sun\\.proxy.*", ".*at com\\.jolbox.*", - "org\\.apache\\.hadoop\\.hive\\.metastore\\.model\\.MConstraint@([0-9]|[a-z])*" + "org\\.apache\\.hadoop\\.hive\\.metastore\\.model\\.MConstraint@([0-9]|[a-z])*", + "(s3.?|swift|wasb.?):\\/\\/[\\w\\.\\/-]*" }); private final Pattern[] partialReservedPlanMask = toPattern(new String[] { @@ -2343,7 +2355,7 @@ public class QTestUtil { int i = 0; while (!statements.isEmpty()) { // PreparedStatement extend Statement - Statement st = (Statement)statements.remove(i); + Statement st = statements.remove(i); try { if (st != null) { st.close();
