ihuzenko commented on a change in pull request #1613: DRILL-6977: Improve Hive tests configuration URL: https://github.com/apache/drill/pull/1613#discussion_r249484133
########## File path: contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/HiveTestFixture.java ########## @@ -0,0 +1,263 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.hive; + +import java.io.File; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; +import java.util.function.Consumer; + +import org.apache.drill.common.exceptions.ErrorHelper; +import org.apache.drill.common.exceptions.ExecutionSetupException; +import org.apache.drill.exec.server.Drillbit; +import org.apache.drill.exec.store.StoragePluginRegistry; +import org.apache.drill.exec.store.hive.HiveStoragePlugin; +import org.apache.drill.exec.store.hive.HiveStoragePluginConfig; +import org.apache.drill.test.BaseDirTestWatcher; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConf.ConfVars; +import org.apache.hadoop.hive.ql.Driver; +import org.apache.hadoop.hive.ql.session.SessionState; + +import static java.util.Objects.nonNull; +import static java.util.Objects.requireNonNull; +import static org.apache.drill.exec.hive.HiveTestUtilities.createDirWithPosixPermissions; + + +/** + * Test fixture for configuration of Hive tests, which + * allows granular control over initialization of test data + * and hive storage plugin. + * <p> + * Below is example of usage HiveTestFixture along with ClusterFixture: + * <p> + * <pre><code> + * // Note that HiveTestFixture don't require extension of ClusterTest, + * // this is just the simplest way for configuring test drillbit + * public class HiveTestExample extends ClusterTest { + * + * private static HiveTestFixture hiveTestFixture; + * + * {@literal @}BeforeClass + * public static void setUp() throws Exception { + * startCluster(ClusterFixture.builder(dirTestWatcher)); + * + * // Below is minimal config which uses defaults from HiveTestFixture.Builder + * // constructor, but any option for driver or storage plugin may be + * // overridden using builder's methods + * hiveTestFixture = HiveTestFixture.builder(dirTestWatcher).build(); + * + * // Use driver manager to configure test data in Hive metastore + * hiveTestFixture.getDriverManager().doWithinSession(HiveTestExample::generateData); + * + * // Use plugin manager to add, remove, update hive storage plugin of one or many test drillbits + * hiveTestFixture.getPluginManager().addHivePluginTo(cluster.drillbits()); + * } + * + * private static void generateData(Driver driver) { + * // Set up data using HiveTestUtilities.executeQuery(driver, sql) + * } + * + * {@literal @}AfterClass + * public static void tearDown() throws Exception { + * if (nonNull(hiveTestFixture)) { + * hiveTestFixture.getPluginManager().removeHivePluginFrom(cluster.drillbits()); + * } + * } + * } + * </code></pre> + */ +public class HiveTestFixture { + + + private final Map<String, String> pluginConf; + + private final Map<String, String> driverConf; + + private final String pluginName; + + private final HivePluginManager pluginManager; + + private final HiveDriverManager driverManager; + + private HiveTestFixture(Builder builder) { + this.pluginConf = new HashMap<>(builder.pluginConf); + this.driverConf = new HashMap<>(builder.driverConf); + this.pluginName = builder.pluginName; + this.pluginManager = new HivePluginManager(); + this.driverManager = new HiveDriverManager(); + } + + public static Builder builder(BaseDirTestWatcher dirWatcher) { + return builder(requireNonNull(dirWatcher, "Parameter 'dirWatcher' can't be null!").getRootDir()); + } + + public static Builder builder(File baseDir) { + return new Builder(requireNonNull(baseDir, "Parameter 'baseDir' can't be null!")); + } + + public HivePluginManager getPluginManager() { + return pluginManager; + } + + public HiveDriverManager getDriverManager() { + return driverManager; + } + + public String getWarehouseDir() { + String warehouseDir = pluginConf.get(ConfVars.METASTOREWAREHOUSE.varname); + return nonNull(warehouseDir) ? warehouseDir : driverConf.get(ConfVars.METASTOREWAREHOUSE.varname); + } + + public class HivePluginManager { + + private HivePluginManager() { + } + + public void addHivePluginTo(Drillbit... drillbits) { + addHivePluginTo(Arrays.asList(drillbits)); + } + + public void addHivePluginTo(Iterable<Drillbit> drillbits) { + try { + for (Drillbit drillbit : drillbits) { + HiveStoragePluginConfig pluginConfig = new HiveStoragePluginConfig(new HashMap<>(pluginConf)); + pluginConfig.setEnabled(true); + drillbit.getContext().getStorage().createOrUpdate(pluginName, pluginConfig, true); + } + } catch (ExecutionSetupException e) { + ErrorHelper.sneakyThrow(e); + } + } + + public void removeHivePluginFrom(Drillbit... drillbits) { + removeHivePluginFrom(Arrays.asList(drillbits)); + } + + public void removeHivePluginFrom(Iterable<Drillbit> drillbits) { + drillbits.forEach(bit -> bit.getContext().getStorage().deletePlugin(pluginName)); + } + + public void updateHivePlugin(Iterable<Drillbit> drillbits, + Map<String, String> configOverride) { + try { + for (Drillbit drillbit : drillbits) { + StoragePluginRegistry pluginRegistry = drillbit.getContext().getStorage(); + HiveStoragePlugin storagePlugin = Objects.requireNonNull( + (HiveStoragePlugin) pluginRegistry.getPlugin(pluginName), + String.format("Hive storage plugin with name '%s' doesn't exist.", pluginName)); + + HiveStoragePluginConfig newPluginConfig = storagePlugin.getConfig(); + newPluginConfig.getConfigProps().putAll(configOverride); + pluginRegistry.createOrUpdate(pluginName, newPluginConfig, true); + } + } catch (ExecutionSetupException e) { + ErrorHelper.sneakyThrow(e); + } + } + + } + + public class HiveDriverManager { + + private HiveDriverManager() { + } + + public void doWithinSession(Consumer<Driver> driverConsumer) { Review comment: Usually this method will be invoked like ```HIVE_TEST_FIXTURE.getDriverManager().doWithinSession(dataGenerator::generateData)``` . I expect that users will create static methods with names similar to ```generateData(Driver driver)``` and pass their references to this one. I agree that current name isn't the best one, but ```setup``` method also won't have a fantastic look. I'd really appreciate if you could suggest another better name for this method :) ---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
