Repository: hive Updated Branches: refs/heads/master 14b972e96 -> d05b7cf09
http://git-wip-us.apache.org/repos/asf/hive/blob/d05b7cf0/standalone-metastore/metastore-tools/tools-common/src/main/java/org/apache/hadoop/hive/metastore/tools/MicroBenchmark.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-tools/tools-common/src/main/java/org/apache/hadoop/hive/metastore/tools/MicroBenchmark.java b/standalone-metastore/metastore-tools/tools-common/src/main/java/org/apache/hadoop/hive/metastore/tools/MicroBenchmark.java new file mode 100644 index 0000000..7b1c8df --- /dev/null +++ b/standalone-metastore/metastore-tools/tools-common/src/main/java/org/apache/hadoop/hive/metastore/tools/MicroBenchmark.java @@ -0,0 +1,123 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.metastore.tools; + +import org.apache.commons.math3.stat.descriptive.DescriptiveStatistics; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +/** + * Micro-benchmark some piece of code.<p> + * + * Every benchmark has three parts: + * <ul> + * <li>Optional pre-test</li> + * <li>Mandatory test</lI> + * <li>Optional post-test</li> + * </ul> + * Measurement consists of the warm-up phase and measurement phase. + * Consumer can specify number of times the warmup and measurement is repeated.<p> + * All time is measured in nanoseconds. + */ +class MicroBenchmark { + // Specify defaults + private static final int WARMUP_DEFAULT = 15; + private static final int ITERATIONS_DEFAULT = 100; + private static final int SCALE_DEFAULT = 1; + + private final int warmup; + private final int iterations; + private final int scaleFactor; + + /** + * Create default micro benchmark measurer + */ + public MicroBenchmark() { + this(WARMUP_DEFAULT, ITERATIONS_DEFAULT, SCALE_DEFAULT); + } + + /** + * Create micro benchmark measurer. + * @param warmup number of test calls for warmup + * @param iterations number of test calls for measurement + */ + MicroBenchmark(int warmup, int iterations) { + this(warmup, iterations, SCALE_DEFAULT); + } + + /** + * Create micro benchmark measurer. + * + * @param warmup number of test calls for warmup + * @param iterations number of test calls for measurement + * @param scaleFactor Every delta is divided by scale factor + */ + private MicroBenchmark(int warmup, int iterations, int scaleFactor) { + this.warmup = warmup; + this.iterations = iterations; + this.scaleFactor = scaleFactor; + } + + /** + * Run the benchmark and measure run-time statistics in nanoseconds.<p> + * Before the run the warm-up phase is executed. + * @param pre Optional pre-test setup + * @param test Mandatory test + * @param post Optional post-test cleanup + * @return Statistics describing the results. All times are in nanoseconds. + */ + public DescriptiveStatistics measure(@Nullable Runnable pre, + @NotNull Runnable test, + @Nullable Runnable post) { + // Warmup phase + for (int i = 0; i < warmup; i++) { + if (pre != null) { + pre.run(); + } + test.run(); + if (post != null) { + post.run(); + } + } + // Run the benchmark + DescriptiveStatistics stats = new DescriptiveStatistics(); + for (int i = 0; i < iterations; i++) { + if (pre != null) { + pre.run(); + } + long start = System.nanoTime(); + test.run(); + long end = System.nanoTime(); + stats.addValue((double)(end - start) / scaleFactor); + if (post != null) { + post.run(); + } + } + return stats; + } + + /** + * Run the benchmark and measure run-time statistics in nanoseconds.<p> + * Before the run the warm-up phase is executed. No pre or post operations are executed. + * @param test test to measure + * @return Statistics describing the results. All times are in nanoseconds. + */ + public DescriptiveStatistics measure(@NotNull Runnable test) { + return measure(null, test, null); + } +} http://git-wip-us.apache.org/repos/asf/hive/blob/d05b7cf0/standalone-metastore/metastore-tools/tools-common/src/main/java/org/apache/hadoop/hive/metastore/tools/Util.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-tools/tools-common/src/main/java/org/apache/hadoop/hive/metastore/tools/Util.java b/standalone-metastore/metastore-tools/tools-common/src/main/java/org/apache/hadoop/hive/metastore/tools/Util.java new file mode 100644 index 0000000..101d675 --- /dev/null +++ b/standalone-metastore/metastore-tools/tools-common/src/main/java/org/apache/hadoop/hive/metastore/tools/Util.java @@ -0,0 +1,554 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.metastore.tools; + +import com.google.common.base.Joiner; +import com.google.common.net.HostAndPort; +import org.apache.hadoop.hive.metastore.TableType; +import org.apache.hadoop.hive.metastore.api.Database; +import org.apache.hadoop.hive.metastore.api.FieldSchema; +import org.apache.hadoop.hive.metastore.api.Partition; +import org.apache.hadoop.hive.metastore.api.PrincipalType; +import org.apache.hadoop.hive.metastore.api.SerDeInfo; +import org.apache.hadoop.hive.metastore.api.StorageDescriptor; +import org.apache.hadoop.hive.metastore.api.Table; +import org.apache.thrift.TException; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.net.URI; +import java.net.URISyntaxException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.regex.Pattern; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +/** + * Helper utilities. The Util class is just a placeholder for static methods, + * it should be never instantiated. + */ +public final class Util { + private static final String DEFAULT_TYPE = "string"; + private static final String TYPE_SEPARATOR = ":"; + private static final String THRIFT_SCHEMA = "thrift"; + static final String DEFAULT_HOST = "localhost"; + private static final String ENV_SERVER = "HMS_HOST"; + private static final String ENV_PORT = "HMS_PORT"; + private static final String PROP_HOST = "hms.host"; + private static final String PROP_PORT = "hms.port"; + + private static final String HIVE_INPUT_FORMAT = "org.apache.hadoop.hive.ql.io.HiveInputFormat"; + private static final String HIVE_OUTPUT_FORMAT = "org.apache.hadoop.hive.ql.io.HiveOutputFormat"; + private static final String LAZY_SIMPLE_SERDE = "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"; + + private static final Pattern[] EMPTY_PATTERN = new Pattern[]{}; + private static final Pattern[] MATCH_ALL_PATTERN = new Pattern[]{Pattern.compile(".*")}; + + private static final Logger LOG = LoggerFactory.getLogger(Util.class); + + // Disable public constructor + private Util() { + } + + /** + * Wrapper that moves all checked exceptions to RuntimeException. + * + * @param throwingSupplier Supplier that throws Exception + * @param <T> Supplier return type + * @return Supplier that throws unchecked exception + */ + public static <T> T throwingSupplierWrapper(ThrowingSupplier<T, Exception> throwingSupplier) { + try { + return throwingSupplier.get(); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + /** + * Version of the Supplier that can throw exceptions. + * + * @param <T> Supplier return type + * @param <E> Exception type + */ + @FunctionalInterface + public interface ThrowingSupplier<T, E extends Exception> { + T get() throws E; + } + + /** + * A builder for Database. The name of the new database is required. Everything else + * selects reasonable defaults. + * This is a modified version of Hive 3.0 DatabaseBuilder. + */ + public static class DatabaseBuilder { + private String name; + private String description; + private String location; + private String ownerName; + private PrincipalType ownerType; + private Map<String, String> params = null; + + // Disable default constructor + private DatabaseBuilder() { + } + + /** + * Constructor from database name. + * + * @param name Database name + */ + public DatabaseBuilder(@NotNull String name) { + this.name = name; + ownerType = PrincipalType.USER; + } + + /** + * Add database description. + * + * @param description Database description string. + * @return this + */ + public DatabaseBuilder withDescription(@NotNull String description) { + this.description = description; + return this; + } + + /** + * Add database location + * + * @param location Database location string + * @return this + */ + public DatabaseBuilder withLocation(@NotNull String location) { + this.location = location; + return this; + } + + /** + * Add Database parameters + * + * @param params database parameters + * @return this + */ + public DatabaseBuilder withParams(@NotNull Map<String, String> params) { + this.params = params; + return this; + } + + /** + * Add a single database parameter. + * + * @param key parameter key + * @param val parameter value + * @return this + */ + public DatabaseBuilder withParam(@NotNull String key, @NotNull String val) { + if (this.params == null) { + this.params = new HashMap<>(); + } + this.params.put(key, val); + return this; + } + + /** + * Add database owner name + * + * @param ownerName new owner name + * @return this + */ + public DatabaseBuilder withOwnerName(@NotNull String ownerName) { + this.ownerName = ownerName; + return this; + } + + /** + * Add owner tyoe + * + * @param ownerType database owner type (USER or GROUP) + * @return this + */ + public DatabaseBuilder withOwnerType(PrincipalType ownerType) { + this.ownerType = ownerType; + return this; + } + + /** + * Build database object + * + * @return database + */ + public Database build() { + Database db = new Database(name, description, location, params); + if (ownerName != null) { + db.setOwnerName(ownerName); + } + if (ownerType != null) { + db.setOwnerType(ownerType); + } + return db; + } + } + + /** + * Builder for Table. + */ + public static class TableBuilder { + private final String dbName; + private final String tableName; + private TableType tableType = TableType.MANAGED_TABLE; + private String location; + private String serde = LAZY_SIMPLE_SERDE; + private String owner; + private List<FieldSchema> columns; + private List<FieldSchema> partitionKeys; + private String inputFormat = HIVE_INPUT_FORMAT; + private String outputFormat = HIVE_OUTPUT_FORMAT; + private Map<String, String> parameters = new HashMap<>(); + + private TableBuilder() { + dbName = null; + tableName = null; + } + + TableBuilder(String dbName, String tableName) { + this.dbName = dbName; + this.tableName = tableName; + } + + static Table buildDefaultTable(String dbName, String tableName) { + return new TableBuilder(dbName, tableName).build(); + } + + TableBuilder withType(TableType tabeType) { + this.tableType = tabeType; + return this; + } + + TableBuilder withOwner(String owner) { + this.owner = owner; + return this; + } + + TableBuilder withColumns(List<FieldSchema> columns) { + this.columns = columns; + return this; + } + + TableBuilder withPartitionKeys(List<FieldSchema> partitionKeys) { + this.partitionKeys = partitionKeys; + return this; + } + + TableBuilder withSerde(String serde) { + this.serde = serde; + return this; + } + + TableBuilder withInputFormat(String inputFormat) { + this.inputFormat = inputFormat; + return this; + } + + TableBuilder withOutputFormat(String outputFormat) { + this.outputFormat = outputFormat; + return this; + } + + TableBuilder withParameter(String name, String value) { + parameters.put(name, value); + return this; + } + + TableBuilder withLocation(String location) { + this.location = location; + return this; + } + + Table build() { + StorageDescriptor sd = new StorageDescriptor(); + if (columns == null) { + sd.setCols(Collections.emptyList()); + } else { + sd.setCols(columns); + } + SerDeInfo serdeInfo = new SerDeInfo(); + serdeInfo.setSerializationLib(serde); + serdeInfo.setName(tableName); + sd.setSerdeInfo(serdeInfo); + sd.setInputFormat(inputFormat); + sd.setOutputFormat(outputFormat); + if (location != null) { + sd.setLocation(location); + } + + Table table = new Table(); + table.setDbName(dbName); + table.setTableName(tableName); + table.setSd(sd); + table.setParameters(parameters); + table.setOwner(owner); + if (partitionKeys != null) { + table.setPartitionKeys(partitionKeys); + } + table.setTableType(tableType.toString()); + return table; + } + } + + /** + * Builder of partitions. + */ + public static class PartitionBuilder { + private final Table table; + private List<String> values; + private String location; + private Map<String, String> parameters = new HashMap<>(); + + private PartitionBuilder() { + table = null; + } + + PartitionBuilder(Table table) { + this.table = table; + } + + PartitionBuilder withValues(List<String> values) { + this.values = new ArrayList<>(values); + return this; + } + + PartitionBuilder withLocation(String location) { + this.location = location; + return this; + } + + PartitionBuilder withParameter(String name, String value) { + parameters.put(name, value); + return this; + } + + PartitionBuilder withParameters(Map<String, String> params) { + parameters = params; + return this; + } + + Partition build() { + Partition partition = new Partition(); + List<String> partitionNames = table.getPartitionKeys() + .stream() + .map(FieldSchema::getName) + .collect(Collectors.toList()); + if (partitionNames.size() != values.size()) { + throw new RuntimeException("Partition values do not match table schema"); + } + List<String> spec = IntStream.range(0, values.size()) + .mapToObj(i -> partitionNames.get(i) + "=" + values.get(i)) + .collect(Collectors.toList()); + + partition.setDbName(table.getDbName()); + partition.setTableName(table.getTableName()); + partition.setParameters(parameters); + partition.setValues(values); + partition.setSd(table.getSd().deepCopy()); + if (this.location == null) { + partition.getSd().setLocation(table.getSd().getLocation() + "/" + Joiner.on("/").join(spec)); + } else { + partition.getSd().setLocation(location); + } + return partition; + } + } + + /** + * Create table schema from parameters. + * + * @param params list of parameters. Each parameter can be either a simple name or + * name:type for non-String types. + * @return table schema description + */ + public static List<FieldSchema> createSchema(@Nullable List<String> params) { + if (params == null || params.isEmpty()) { + return Collections.emptyList(); + } + + return params.stream() + .map(Util::param2Schema) + .collect(Collectors.toList()); + } + + /** + * Get server URI.<p> + * HMS host is obtained from + * <ol> + * <li>Argument</li> + * <li>HMS_HOST environment parameter</li> + * <li>hms.host Java property</li> + * <li>use 'localhost' if above fails</li> + * </ol> + * HMS Port is obtained from + * <ol> + * <li>Argument</li> + * <li>host:port string</li> + * <li>HMS_PORT environment variable</li> + * <li>hms.port Java property</li> + * <li>default port value</li> + * </ol> + * + * @param host HMS host string. + * @param portString HMS port + * @return HMS URI + * @throws URISyntaxException if URI is is invalid + */ + public static @Nullable URI getServerUri(@Nullable String host, @Nullable String portString) throws + URISyntaxException { + if (host == null) { + host = System.getenv(ENV_SERVER); + } + if (host == null) { + host = System.getProperty(PROP_HOST); + } + if (host == null) { + host = DEFAULT_HOST; + } + host = host.trim(); + + if ((portString == null || portString.isEmpty() || portString.equals("0")) && + !host.contains(":")) { + portString = System.getenv(ENV_PORT); + if (portString == null) { + portString = System.getProperty(PROP_PORT); + } + } + Integer port = Constants.HMS_DEFAULT_PORT; + if (portString != null) { + port = Integer.parseInt(portString); + } + + HostAndPort hp = HostAndPort.fromString(host) + .withDefaultPort(port); + + LOG.info("Connecting to {}:{}", hp.getHostText(), hp.getPort()); + + return new URI(THRIFT_SCHEMA, null, hp.getHostText(), hp.getPort(), + null, null, null); + } + + + private static FieldSchema param2Schema(@NotNull String param) { + String colType = DEFAULT_TYPE; + String name = param; + if (param.contains(TYPE_SEPARATOR)) { + String[] parts = param.split(TYPE_SEPARATOR); + name = parts[0]; + colType = parts[1].toLowerCase(); + } + return new FieldSchema(name, colType, ""); + } + + /** + * Create multiple partition objects. + * + * @param table + * @param arguments - list of partition names. + * @param npartitions - Partition parameters + * @return List of created partitions + */ + static List<Partition> createManyPartitions(@NotNull Table table, + @Nullable Map<String, String> parameters, + @NotNull List<String> arguments, + int npartitions) { + return IntStream.range(0, npartitions) + .mapToObj(i -> + new PartitionBuilder(table) + .withParameters(parameters) + .withValues( + arguments.stream() + .map(a -> a + i) + .collect(Collectors.toList())).build()) + .collect(Collectors.toList()); + } + + /** + * Add many partitions in one HMS call + * + * @param client HMS Client + * @param dbName database name + * @param tableName table name + * @param arguments list of partition names + * @param npartitions number of partitions to create + * @throws TException if fails to create partitions + */ + static Object addManyPartitions(@NotNull HMSClient client, + @NotNull String dbName, + @NotNull String tableName, + @Nullable Map<String, String> parameters, + @NotNull List<String> arguments, + int npartitions) throws TException { + Table table = client.getTable(dbName, tableName); + client.addPartitions(createManyPartitions(table, parameters, arguments, npartitions)); + return null; + } + + static List<String> generatePartitionNames(@NotNull String prefix, int npartitions) { + return IntStream.range(0, npartitions).mapToObj(i -> prefix + i).collect(Collectors.toList()); + } + + static void addManyPartitionsNoException(@NotNull HMSClient client, + @NotNull String dbName, + @NotNull String tableName, + @Nullable Map<String, String> parameters, + List<String> arguments, + int npartitions) { + throwingSupplierWrapper(() -> + addManyPartitions(client, dbName, tableName, parameters, arguments, npartitions)); + } + + /** + * Filter candidates - find all that match positive matches and do not match + * any negative matches. + * + * @param candidates list of candidate strings. If null, return an empty list. + * @param positivePatterns list of regexp that should all match. If null, everything matches. + * @param negativePatterns list of regexp, none of these should match. If null, everything matches. + * @return list of filtered results. + */ + public static List<String> filterMatches(@Nullable List<String> candidates, + @Nullable Pattern[] positivePatterns, + @Nullable Pattern[] negativePatterns) { + if (candidates == null || candidates.isEmpty()) { + return Collections.emptyList(); + } + final Pattern[] positive = (positivePatterns == null || positivePatterns.length == 0) ? + MATCH_ALL_PATTERN : positivePatterns; + final Pattern[] negative = negativePatterns == null ? EMPTY_PATTERN : negativePatterns; + + return candidates.stream() + .filter(c -> Arrays.stream(positive).anyMatch(p -> p.matcher(c).matches())) + .filter(c -> Arrays.stream(negative).noneMatch(p -> p.matcher(c).matches())) + .collect(Collectors.toList()); + } +} http://git-wip-us.apache.org/repos/asf/hive/blob/d05b7cf0/standalone-metastore/metastore-tools/tools-common/src/test/java/org/apache/hadoop/hive/metastore/tools/HMSClientTest.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-tools/tools-common/src/test/java/org/apache/hadoop/hive/metastore/tools/HMSClientTest.java b/standalone-metastore/metastore-tools/tools-common/src/test/java/org/apache/hadoop/hive/metastore/tools/HMSClientTest.java new file mode 100644 index 0000000..ab4b625 --- /dev/null +++ b/standalone-metastore/metastore-tools/tools-common/src/test/java/org/apache/hadoop/hive/metastore/tools/HMSClientTest.java @@ -0,0 +1,206 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.metastore.tools; + +import com.google.common.collect.ImmutableMap; +import org.apache.hadoop.hive.metastore.api.AlreadyExistsException; +import org.apache.hadoop.hive.metastore.api.Database; +import org.apache.hadoop.hive.metastore.api.InvalidObjectException; +import org.apache.hadoop.hive.metastore.api.MetaException; +import org.apache.hadoop.hive.metastore.api.NoSuchObjectException; +import org.apache.hadoop.hive.metastore.api.Table; +import org.apache.hadoop.hive.metastore.tools.HMSClient; +import org.apache.hadoop.hive.metastore.tools.Util; +import org.apache.thrift.TException; +import org.hamcrest.MatcherAssert; +import org.hamcrest.Matchers; +import org.junit.Assume; +import org.junit.Before; +import org.junit.Test; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; + +import java.util.Set; + +import static org.apache.hadoop.hive.metastore.tools.Util.getServerUri; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.equalToIgnoringCase; +import static org.hamcrest.Matchers.hasItem; +import static org.junit.Assume.assumeTrue; +import static org.junit.jupiter.api.Assertions.assertThrows; + +public class HMSClientTest { + private static final String PARAM_NAME = "param"; + private static final String VALUE_NAME = "value"; + private static final String TEST_DATABASE="hmsClientTest"; + private static final String TEST_DATABASE_DESCRIPTION="hmsclienttest description"; + private static final ImmutableMap<String, String> TEST_DATABASE_PARAMS = + new ImmutableMap.Builder<String, String>() + .put(PARAM_NAME, VALUE_NAME) + .build(); + private static boolean hasClient = false; + + private static final String TEST_TABLE_NAME="test1"; + private static final Table TEST_TABLE = + Util.TableBuilder.buildDefaultTable(TEST_DATABASE, TEST_TABLE_NAME); + + private static HMSClient client = null; + + @BeforeAll + static void init() throws Exception { + Database db = new Util.DatabaseBuilder(TEST_DATABASE) + .withDescription(TEST_DATABASE_DESCRIPTION) + .withParams(TEST_DATABASE_PARAMS) + .build(); + // Create client and default test database + try { + client = + new HMSClient(getServerUri(null, null), null); + client.createDatabase(db); + } catch (Exception e) { + System.out.println(e.getMessage()); + e.printStackTrace(); + } + } + + @AfterAll + static void shutdown() throws TException { + if (client != null) { + // Destroy test database + client.dropDatabase(TEST_DATABASE); + } + } + + @Before + public void beforeTest() { + Assume.assumeTrue(client != null); + } + + /** + * Verify that list of databases contains "default" and test database + * @throws Exception + */ + @Test + public void getAllDatabases() throws Exception { + Set<String> databases = client.getAllDatabases(null); + MatcherAssert.assertThat(databases, Matchers.hasItem("default")); + MatcherAssert.assertThat(databases, Matchers.hasItem(TEST_DATABASE.toLowerCase())); + assertThat(client.getAllDatabases(TEST_DATABASE.toLowerCase()), Matchers.contains(TEST_DATABASE.toLowerCase())); + } + + /** + * Verify that an attempt to create an existing database throws AlreadyExistsException. + */ + @Test + public void createExistingDatabase() { + Throwable exception = Assertions.assertThrows(AlreadyExistsException.class, + () -> client.createDatabase(TEST_DATABASE)); + } + + /** + * Creating a database with null name should not be allowed + * and should throw MetaException. + */ + @Test + public void createDatabaseNullName() { + Database db = new Util.DatabaseBuilder(TEST_DATABASE) + .build(); + db.setName(null); + Throwable exception = Assertions.assertThrows(MetaException.class, + () -> client.createDatabase(db)); + } + + /** + * Creating a database with an empty name should not be allowed + * and should throw InvalidObjectException + */ + @Test + public void createDatabaseEmptyName() { + Assume.assumeTrue(client != null); + Database db = new Util.DatabaseBuilder(TEST_DATABASE) + .build(); + db.setName(""); + Throwable exception = Assertions.assertThrows(InvalidObjectException.class, + () -> client.createDatabase(db)); + } + + /** + * Verify that getDatabase() returns all expected fields + * @throws TException if fails to get database info + */ + @Test + public void getDatabase() throws TException { + Database db = client.getDatabase(TEST_DATABASE); + MatcherAssert.assertThat(db.getName(), Matchers.equalToIgnoringCase(TEST_DATABASE)); + MatcherAssert.assertThat(db.getDescription(), Matchers.equalTo(TEST_DATABASE_DESCRIPTION)); + MatcherAssert.assertThat(db.getParameters(), Matchers.equalTo(TEST_DATABASE_PARAMS)); + MatcherAssert.assertThat(db.getLocationUri(), Matchers.containsString(TEST_DATABASE.toLowerCase())); + } + + /** + * Verify that locating database is case-insensitive + */ + @Test + public void getDatabaseCI() throws TException { + Database db = client.getDatabase(TEST_DATABASE.toUpperCase()); + MatcherAssert.assertThat(db.getName(), Matchers.equalToIgnoringCase(TEST_DATABASE)); + MatcherAssert.assertThat(db.getDescription(), Matchers.equalTo(TEST_DATABASE_DESCRIPTION)); + MatcherAssert.assertThat(db.getParameters(), Matchers.equalTo(TEST_DATABASE_PARAMS)); + MatcherAssert.assertThat(db.getLocationUri(), Matchers.containsString(TEST_DATABASE.toLowerCase())); + } + + /** + * Verify that searching for non-existing database throws + * NoSuchObjectException + */ + @Test + public void getNonExistingDb() { + Throwable exception = Assertions.assertThrows(NoSuchObjectException.class, + () -> client.getDatabase("WhatIsThisDatabase")); + } + + + /** + * Verify that dropping for non-existing database throws + * NoSuchObjectException + */ + @Test + public void dropNonExistingDb() { + Throwable exception = Assertions.assertThrows(NoSuchObjectException.class, + () -> client.dropDatabase("WhatIsThisDatabase")); + } + + @Test + public void getAllTables() throws TException { + try { + client.createTable(TEST_TABLE); + assertThat(client.getAllTables(TEST_DATABASE, null), Matchers.contains(TEST_TABLE_NAME)); + } catch (Exception e) { + System.out.println(e.getMessage()); + e.printStackTrace(); + } finally { + client.dropTable(TEST_DATABASE, TEST_TABLE_NAME); + } + } + +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hive/blob/d05b7cf0/standalone-metastore/metastore-tools/tools-common/src/test/java/org/apache/hadoop/hive/metastore/tools/UtilTest.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-tools/tools-common/src/test/java/org/apache/hadoop/hive/metastore/tools/UtilTest.java b/standalone-metastore/metastore-tools/tools-common/src/test/java/org/apache/hadoop/hive/metastore/tools/UtilTest.java new file mode 100644 index 0000000..202979d --- /dev/null +++ b/standalone-metastore/metastore-tools/tools-common/src/test/java/org/apache/hadoop/hive/metastore/tools/UtilTest.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.metastore.tools; + +import com.google.common.collect.ImmutableList; +import org.junit.Test; + +import java.util.Collections; +import java.util.List; +import java.util.regex.Pattern; + +import static org.apache.hadoop.hive.metastore.tools.Util.filterMatches; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.core.Is.is; + +public class UtilTest { + + public UtilTest() { + } + + /** + * Test that a null pattern returns all candidates for iflterMatches. + * Also verify that null candidates result in an empty result list. + */ + @Test + public void filterMatchesEmpty() { + List<String> candidates = ImmutableList.of("foo", "bar"); + assertThat(filterMatches(candidates, null, null), is(candidates)); + assertThat(filterMatches(null, null, null), is(Collections.emptyList())); + } + + /** + * Test positive matches when some candidates match. + */ + @Test + public void filterMatchesPositive() { + List<String> candidates = ImmutableList.of("foo", "bar"); + List<String> expected = ImmutableList.of("foo"); + assertThat(filterMatches(candidates, new Pattern[]{Pattern.compile("f.*")}, null), + is(expected)); + } + + /** + * Test negative matches + */ + @Test + public void filterMatchesNegative() { + List<String> candidates = ImmutableList.of("a", "b"); + List<String> expected = ImmutableList.of("a"); + assertThat(filterMatches(candidates, null, new Pattern[]{Pattern.compile("b")}), + is(expected)); + } + + /** + * Test that multiple patterns are handled correctly. We use one positive and one negative parrent. + */ + @Test + public void filterMatchesMultiple() { + List<String> candidates = ImmutableList.of("a", "b", "any", "boom", "hello"); + List<String> patterns = ImmutableList.of("^a", "!y$"); + List<String> expected = ImmutableList.of("a"); + assertThat(filterMatches(candidates, new Pattern[]{Pattern.compile("^a")}, new Pattern[]{Pattern.compile("y$")}), + is(expected)); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hive/blob/d05b7cf0/standalone-metastore/pom.xml ---------------------------------------------------------------------- diff --git a/standalone-metastore/pom.xml b/standalone-metastore/pom.xml index 99bf437..ee3daed 100644 --- a/standalone-metastore/pom.xml +++ b/standalone-metastore/pom.xml @@ -19,6 +19,7 @@ <modules> <module>metastore-common</module> <module>metastore-server</module> + <module>metastore-tools</module> </modules> <parent> <groupId>org.apache</groupId> @@ -91,6 +92,7 @@ <protobuf.version>2.5.0</protobuf.version> <sqlline.version>1.3.0</sqlline.version> <storage-api.version>2.7.0-SNAPSHOT</storage-api.version> + <hamcrest.version>1.3</hamcrest.version> <!-- Thrift properties --> <thrift.home>you-must-set-this-to-run-thrift</thrift.home> @@ -300,6 +302,13 @@ <version>9.3-1102-jdbc41</version> <scope>test</scope> </dependency> + <!-- https://mvnrepository.com/artifact/org.hamcrest/hamcrest-all --> + <dependency> + <groupId>org.hamcrest</groupId> + <artifactId>hamcrest-all</artifactId> + <version>${hamcrest.version}</version> + <scope>test</scope> + </dependency> </dependencies> </dependencyManagement>
