Repository: hive Updated Branches: refs/heads/master fde503dca -> d9801d9c6
http://git-wip-us.apache.org/repos/asf/hive/blob/d9801d9c/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestPartitionNameWhitelistValidation.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestPartitionNameWhitelistValidation.java b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestPartitionNameWhitelistValidation.java new file mode 100644 index 0000000..180a666 --- /dev/null +++ b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestPartitionNameWhitelistValidation.java @@ -0,0 +1,122 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.metastore; + +import static org.junit.Assert.*; + +import java.util.ArrayList; +import java.util.List; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.metastore.conf.MetastoreConf; +import org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars; +import org.junit.BeforeClass; +import org.junit.Test; + +// Validate the metastore client call validatePartitionNameCharacters to ensure it throws +// an exception if partition fields contain Unicode characters or commas + +public class TestPartitionNameWhitelistValidation { + + private static final String partitionValidationPattern = "[\\x20-\\x7E&&[^,]]*"; + private static Configuration conf; + private static HiveMetaStoreClient msc; + + @BeforeClass + public static void setupBeforeClass() throws Exception { + System.setProperty(ConfVars.PARTITION_NAME_WHITELIST_PATTERN.toString(), partitionValidationPattern); + conf = MetastoreConf.newMetastoreConf(); + MetaStoreTestUtils.setConfForStandloneMode(conf); + msc = new HiveMetaStoreClient(conf); + } + + // Runs an instance of DisallowUnicodePreEventListener + // Returns whether or not it succeeded + private boolean runValidation(List<String> partVals) { + try { + msc.validatePartitionNameCharacters(partVals); + } catch (Exception e) { + return false; + } + + return true; + } + + // Sample data + private List<String> getPartValsWithUnicode() { + List<String> partVals = new ArrayList<>(); + partVals.add("klâwen"); + partVals.add("tägelîch"); + + return partVals; + } + + private List<String> getPartValsWithCommas() { + List<String> partVals = new ArrayList<>(); + partVals.add("a,b"); + partVals.add("c,d,e,f"); + + return partVals; + } + + private List<String> getPartValsWithValidCharacters() { + List<String> partVals = new ArrayList<>(); + partVals.add("part1"); + partVals.add("part2"); + + return partVals; + } + + @Test + public void testAddPartitionWithCommas() { + assertFalse("Add a partition with commas in name", + runValidation(getPartValsWithCommas())); + } + + @Test + public void testAddPartitionWithUnicode() { + assertFalse("Add a partition with unicode characters in name", + runValidation(getPartValsWithUnicode())); + } + + @Test + public void testAddPartitionWithValidPartVal() { + assertTrue("Add a partition with unicode characters in name", + runValidation(getPartValsWithValidCharacters())); + } + + @Test + public void testAppendPartitionWithUnicode() { + assertFalse("Append a partition with unicode characters in name", + runValidation(getPartValsWithUnicode())); + } + + @Test + public void testAppendPartitionWithCommas() { + assertFalse("Append a partition with unicode characters in name", + runValidation(getPartValsWithCommas())); + } + + @Test + public void testAppendPartitionWithValidCharacters() { + assertTrue("Append a partition with no unicode characters in name", + runValidation(getPartValsWithValidCharacters())); + } + +} http://git-wip-us.apache.org/repos/asf/hive/blob/d9801d9c/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestRemoteHiveMetaStore.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestRemoteHiveMetaStore.java b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestRemoteHiveMetaStore.java new file mode 100644 index 0000000..8976474 --- /dev/null +++ b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestRemoteHiveMetaStore.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.metastore; + +import org.apache.hadoop.hive.metastore.conf.MetastoreConf; +import org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars; +import org.apache.hadoop.hive.metastore.security.HadoopThriftAuthBridge; +import org.junit.Assert; +import org.junit.Before; + + +public class TestRemoteHiveMetaStore extends TestHiveMetaStore { + private static boolean isServerStarted = false; + protected static int port; + + public TestRemoteHiveMetaStore() { + super(); + isThriftClient = true; + } + + @Before + public void setUp() throws Exception { + super.setUp(); + + if (isServerStarted) { + Assert.assertNotNull("Unable to connect to the MetaStore server", client); + MetastoreConf.setVar(conf, ConfVars.THRIFT_URIS, "thrift://localhost:" + port); + return; + } + + port = MetaStoreTestUtils.startMetaStoreWithRetry(HadoopThriftAuthBridge.getBridge(), + conf); + System.out.println("Starting MetaStore Server on port " + port); + isServerStarted = true; + + // This is default case with setugi off for both client and server + client = createClient(); + } + + @Override + protected HiveMetaStoreClient createClient() throws Exception { + MetastoreConf.setVar(conf, ConfVars.THRIFT_URIS, "thrift://localhost:" + port); + MetastoreConf.setBoolVar(conf, ConfVars.EXECUTE_SET_UGI, false); + return new HiveMetaStoreClient(conf); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hive/blob/d9801d9c/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestRemoteHiveMetaStoreIpAddress.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestRemoteHiveMetaStoreIpAddress.java b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestRemoteHiveMetaStoreIpAddress.java new file mode 100644 index 0000000..370cd28 --- /dev/null +++ b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestRemoteHiveMetaStoreIpAddress.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.metastore; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.metastore.api.Database; +import org.apache.hadoop.hive.metastore.conf.MetastoreConf; +import org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars; +import org.apache.hadoop.hive.metastore.security.HadoopThriftAuthBridge; +import org.junit.Before; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * + * TestRemoteHiveMetaStoreIpAddress. + * + * Test which checks that the remote Hive metastore stores the proper IP address using + * IpAddressListener + */ +public class TestRemoteHiveMetaStoreIpAddress { + private static final Logger LOG = LoggerFactory.getLogger(TestRemoteHiveMetaStoreIpAddress.class); + private static Configuration conf; + private static HiveMetaStoreClient msc; + + @Before + public void setUp() throws Exception { + conf = MetastoreConf.newMetastoreConf(); + + + System.setProperty(ConfVars.EVENT_LISTENERS.toString(), IpAddressListener.class.getName()); + MetaStoreTestUtils.setConfForStandloneMode(conf); + int port = MetaStoreTestUtils.startMetaStoreWithRetry(HadoopThriftAuthBridge.getBridge(), conf); + LOG.debug("Starting MetaStore Server on port " + port); + MetastoreConf.setVar(conf, ConfVars.THRIFT_URIS, "thrift://localhost:" + port); + + msc = new HiveMetaStoreClient(conf); + } + + @Test + public void testIpAddress() throws Exception { + Database db = new Database(); + db.setName("testIpAddressIp"); + msc.createDatabase(db); + msc.dropDatabase(db.getName()); + } +} http://git-wip-us.apache.org/repos/asf/hive/blob/d9801d9c/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestRemoteUGIHiveMetaStoreIpAddress.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestRemoteUGIHiveMetaStoreIpAddress.java b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestRemoteUGIHiveMetaStoreIpAddress.java new file mode 100644 index 0000000..92d2d0e --- /dev/null +++ b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestRemoteUGIHiveMetaStoreIpAddress.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.metastore; + + +import org.apache.hadoop.hive.metastore.conf.MetastoreConf; + +public class TestRemoteUGIHiveMetaStoreIpAddress extends TestRemoteHiveMetaStoreIpAddress { + public TestRemoteUGIHiveMetaStoreIpAddress() { + System.setProperty(MetastoreConf.ConfVars.EXECUTE_SET_UGI.toString(), "true"); + } + +} http://git-wip-us.apache.org/repos/asf/hive/blob/d9801d9c/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestRetryingHMSHandler.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestRetryingHMSHandler.java b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestRetryingHMSHandler.java new file mode 100644 index 0000000..badcd60 --- /dev/null +++ b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestRetryingHMSHandler.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.metastore; + +import java.util.concurrent.TimeUnit; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.metastore.api.Database; +import org.apache.hadoop.hive.metastore.api.Table; +import org.apache.hadoop.hive.metastore.client.builder.TableBuilder; +import org.apache.hadoop.hive.metastore.conf.MetastoreConf; +import org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars; +import org.apache.hadoop.hive.metastore.security.HadoopThriftAuthBridge; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +/** + * TestRetryingHMSHandler. Test case for + * {@link org.apache.hadoop.hive.metastore.RetryingHMSHandler} + */ +public class TestRetryingHMSHandler { + private Configuration conf; + private HiveMetaStoreClient msc; + + @Before + public void setUp() throws Exception { + System.setProperty("hive.metastore.pre.event.listeners", + AlternateFailurePreListener.class.getName()); + conf = MetastoreConf.newMetastoreConf(); + MetastoreConf.setLongVar(conf, ConfVars.THRIFT_CONNECTION_RETRIES, 3); + MetastoreConf.setLongVar(conf, ConfVars.HMSHANDLERATTEMPTS, 2); + MetastoreConf.setTimeVar(conf, ConfVars.HMSHANDLERINTERVAL, 0, TimeUnit.MILLISECONDS); + MetastoreConf.setBoolVar(conf, ConfVars.HMSHANDLERFORCERELOADCONF, false); + MetaStoreTestUtils.setConfForStandloneMode(conf); + int port = MetaStoreTestUtils.startMetaStoreWithRetry(HadoopThriftAuthBridge.getBridge(), conf); + MetastoreConf.setVar(conf, ConfVars.THRIFT_URIS, "thrift://localhost:" + port); + msc = new HiveMetaStoreClient(conf); + } + + // Create a database and a table in that database. Because the AlternateFailurePreListener is + // being used each attempt to create something should require two calls by the RetryingHMSHandler + @Test + public void testRetryingHMSHandler() throws Exception { + String dbName = "hive4159"; + String tblName = "tmptbl"; + + Database db = new Database(); + db.setName(dbName); + msc.createDatabase(db); + + Assert.assertEquals(2, AlternateFailurePreListener.getCallCount()); + + Table tbl = new TableBuilder() + .setDbName(dbName) + .setTableName(tblName) + .addCol("c1", ColumnType.STRING_TYPE_NAME) + .build(); + + msc.createTable(tbl); + + Assert.assertEquals(4, AlternateFailurePreListener.getCallCount()); + } + +} http://git-wip-us.apache.org/repos/asf/hive/blob/d9801d9c/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestSetUGIOnBothClientServer.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestSetUGIOnBothClientServer.java b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestSetUGIOnBothClientServer.java new file mode 100644 index 0000000..e34d089 --- /dev/null +++ b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestSetUGIOnBothClientServer.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.metastore; + +import org.apache.hadoop.hive.metastore.conf.MetastoreConf; + +public class TestSetUGIOnBothClientServer extends TestRemoteHiveMetaStore{ + + public TestSetUGIOnBothClientServer() { + super(); + isThriftClient = true; + // This will turn on setugi on both client and server processes of the test. + System.setProperty(MetastoreConf.ConfVars.EXECUTE_SET_UGI.toString(), "true"); + } +} http://git-wip-us.apache.org/repos/asf/hive/blob/d9801d9c/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestSetUGIOnOnlyClient.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestSetUGIOnOnlyClient.java b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestSetUGIOnOnlyClient.java new file mode 100644 index 0000000..beff656 --- /dev/null +++ b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestSetUGIOnOnlyClient.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.metastore; + +import org.apache.hadoop.hive.metastore.conf.MetastoreConf; +import org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars; + +public class TestSetUGIOnOnlyClient extends TestRemoteHiveMetaStore{ + + @Override + protected HiveMetaStoreClient createClient() throws Exception { + MetastoreConf.setVar(conf, ConfVars.THRIFT_URIS, "thrift://localhost:" + port); + MetastoreConf.setBoolVar(conf, ConfVars.EXECUTE_SET_UGI, true); + return new HiveMetaStoreClient(conf); + } +} http://git-wip-us.apache.org/repos/asf/hive/blob/d9801d9c/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestSetUGIOnOnlyServer.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestSetUGIOnOnlyServer.java b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestSetUGIOnOnlyServer.java new file mode 100644 index 0000000..bec5a55 --- /dev/null +++ b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestSetUGIOnOnlyServer.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.metastore; + +import org.apache.hadoop.hive.metastore.conf.MetastoreConf; +import org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars; + +public class TestSetUGIOnOnlyServer extends TestSetUGIOnBothClientServer { + + @Override + protected HiveMetaStoreClient createClient() throws Exception { + MetastoreConf.setVar(conf, ConfVars.THRIFT_URIS, "thrift://localhost:" + port); + MetastoreConf.setBoolVar(conf, ConfVars.EXECUTE_SET_UGI, false); + return new HiveMetaStoreClient(conf); + } +} http://git-wip-us.apache.org/repos/asf/hive/blob/d9801d9c/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/cache/TestCachedStore.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/cache/TestCachedStore.java b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/cache/TestCachedStore.java index c0e84fc..b9a8f61 100644 --- a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/cache/TestCachedStore.java +++ b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/cache/TestCachedStore.java @@ -25,6 +25,7 @@ import java.util.Map; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.common.ndv.hll.HyperLogLog; +import org.apache.hadoop.hive.metastore.MetaStoreTestUtils; import org.apache.hadoop.hive.metastore.ObjectStore; import org.apache.hadoop.hive.metastore.TableType; import org.apache.hadoop.hive.metastore.TestObjectStore.MockPartitionExpressionProxy; @@ -60,8 +61,7 @@ public class TestCachedStore { public void setUp() throws Exception { Configuration conf = MetastoreConf.newMetastoreConf(); MetastoreConf.setBoolVar(conf, MetastoreConf.ConfVars.HIVE_IN_TEST, true); - MetastoreConf.setVar(conf, MetastoreConf.ConfVars.EXPRESSION_PROXY_CLASS, - MockPartitionExpressionProxy.class.getName()); + MetaStoreTestUtils.setConfForStandloneMode(conf); objectStore = new ObjectStore(); objectStore.setConf(conf); cachedStore = new CachedStore();
