Repository: hive Updated Branches: refs/heads/master 2f1015365 -> f317d738f
HIVE-15363: Execute hive-blobstore tests using ProxyLocalFileSystem (Sergio Pena, reviewed by Mohit Sabharwal) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/f317d738 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/f317d738 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/f317d738 Branch: refs/heads/master Commit: f317d738f526cf3a43955358c48612fab2cfa681 Parents: 2f10153 Author: Sergio Pena <[email protected]> Authored: Tue Dec 6 14:59:42 2016 -0600 Committer: Sergio Pena <[email protected]> Committed: Tue Dec 6 14:59:42 2016 -0600 ---------------------------------------------------------------------- itests/hive-blobstore/pom.xml | 25 -------------------- .../src/test/resources/hive-site.xml | 19 +++++++++++++++ .../control/AbstractCoreBlobstoreCliDriver.java | 13 +++++++++- .../org/apache/hadoop/hive/ql/QTestUtil.java | 2 +- 4 files changed, 32 insertions(+), 27 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/f317d738/itests/hive-blobstore/pom.xml ---------------------------------------------------------------------- diff --git a/itests/hive-blobstore/pom.xml b/itests/hive-blobstore/pom.xml index a62885f..edc0872 100644 --- a/itests/hive-blobstore/pom.xml +++ b/itests/hive-blobstore/pom.xml @@ -39,31 +39,6 @@ <test.dfs.mkdir>-mkdir -p</test.dfs.mkdir> </properties> - <profiles> - <profile> - <id>tests-off</id> - <activation> - <file> - <missing>src/test/resources/blobstore-conf.xml</missing> - </file> - </activation> - <properties> - <maven.test.skip>true</maven.test.skip> - </properties> - </profile> - <profile> - <id>tests-on</id> - <activation> - <file> - <exists>src/test/resources/blobstore-conf.xml</exists> - </file> - </activation> - <properties> - <maven.test.skip>false</maven.test.skip> - </properties> - </profile> - </profiles> - <dependencies> <!-- dependencies are always listed in sorted order by groupId, artifactId --> <dependency> http://git-wip-us.apache.org/repos/asf/hive/blob/f317d738/itests/hive-blobstore/src/test/resources/hive-site.xml ---------------------------------------------------------------------- diff --git a/itests/hive-blobstore/src/test/resources/hive-site.xml b/itests/hive-blobstore/src/test/resources/hive-site.xml index ffac40a..406b3b8 100644 --- a/itests/hive-blobstore/src/test/resources/hive-site.xml +++ b/itests/hive-blobstore/src/test/resources/hive-site.xml @@ -261,6 +261,25 @@ </property> <!-- + The following 3 properties are used for testing blobstore changes on the local filesystem. + If tests are required to run on an real Blobstore system, then create the blobstore-conf.xml. + --> + <property> + <name>fs.pblob.impl</name> + <value>org.apache.hadoop.fs.ProxyLocalFileSystem</value> + </property> + + <property> + <name>hive.blobstore.supported.schemes</name> + <value>pblob,s3a,s3c,s3</value> + </property> + + <property> + <name>test.blobstore.path</name> + <value>pblob:${system:test.tmp.dir}/bucket</value> + </property> + + <!-- To run these tests: # Create a file blobstore-conf.xml - DO NOT ADD TO REVISION CONTROL # Add the credentials for the service you are testing against http://git-wip-us.apache.org/repos/asf/hive/blob/f317d738/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCoreBlobstoreCliDriver.java ---------------------------------------------------------------------- diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCoreBlobstoreCliDriver.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCoreBlobstoreCliDriver.java index 2f57683..9c97c31 100644 --- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCoreBlobstoreCliDriver.java +++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCoreBlobstoreCliDriver.java @@ -23,7 +23,12 @@ import static org.junit.Assert.fail; import com.google.common.base.Strings; import java.text.SimpleDateFormat; import java.util.Calendar; +import java.util.Map; + import org.apache.hadoop.hive.cli.control.AbstractCliConfig.MetastoreType; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveVariableSource; +import org.apache.hadoop.hive.conf.VariableSubstitution; import org.apache.hadoop.hive.ql.QTestUtil; import org.apache.hadoop.hive.ql.QTestUtil.MiniClusterType; import org.junit.After; @@ -157,7 +162,13 @@ public abstract class AbstractCoreBlobstoreCliDriver extends CliAdapter { * the same ${test.blobstore.path} (possible if test runs are controlled by an automated system) */ private void setupUniqueTestPath() { - String testBlobstorePath = qt.getConf().get(HCONF_TEST_BLOBSTORE_PATH); + String testBlobstorePath = new VariableSubstitution(new HiveVariableSource() { + @Override + public Map<String, String> getHiveVariable() { + return null; + } + }).substitute(new HiveConf(), qt.getConf().get(HCONF_TEST_BLOBSTORE_PATH)); + testBlobstorePath = QTestUtil.ensurePathEndsInSlash(testBlobstorePath); testBlobstorePath += QTestUtil.ensurePathEndsInSlash(this.getClass().getSimpleName()); // name of child class String uid = new SimpleDateFormat("yyyyMMdd.HHmmss.SSS").format(Calendar.getInstance().getTime()) http://git-wip-us.apache.org/repos/asf/hive/blob/f317d738/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java ---------------------------------------------------------------------- diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java index 5bf23e7..8b383c7 100644 --- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java +++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java @@ -1731,7 +1731,7 @@ public class QTestUtil { /* This list may be modified by specific cli drivers to mask strings that change on every test */ private List<Pair<Pattern, String>> patternsWithMaskComments = new ArrayList<Pair<Pattern, String>>() {{ - add(toPatternPair("(s3.?|swift|wasb.?).*hive-staging.*","### BLOBSTORE_STAGING_PATH ###")); + add(toPatternPair("(pblob|s3.?|swift|wasb.?).*hive-staging.*","### BLOBSTORE_STAGING_PATH ###")); }}; private Pair<Pattern, String> toPatternPair(String patternStr, String maskComment) {
