Repository: incubator-drill Updated Branches: refs/heads/master 0a3ae106e -> 699851b8d
DRILL-1112: UI changes to storage plugins only take effect after a drillbit restart. Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/8289f726 Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/8289f726 Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/8289f726 Branch: refs/heads/master Commit: 8289f726cf8d8407e8bcf7c797f301f4fc2950bd Parents: 0a3ae10 Author: Aditya Kishore <adi...@maprtech.com> Authored: Fri Jul 11 21:48:00 2014 -0700 Committer: Aditya Kishore <adi...@maprtech.com> Committed: Sat Jul 12 17:25:34 2014 -0700 ---------------------------------------------------------------------- distribution/src/assemble/bin.xml | 7 +++ .../exec/server/rest/StorageResources.java | 33 ++++++++++---- .../drill/exec/store/StoragePluginRegistry.java | 47 ++++++++++++++------ .../src/main/resources/rest/storage/list.ftl | 37 ++++++++++----- .../src/main/resources/rest/storage/update.ftl | 2 +- 5 files changed, 92 insertions(+), 34 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8289f726/distribution/src/assemble/bin.xml ---------------------------------------------------------------------- diff --git a/distribution/src/assemble/bin.xml b/distribution/src/assemble/bin.xml index bc8fc5f..f66544a 100644 --- a/distribution/src/assemble/bin.xml +++ b/distribution/src/assemble/bin.xml @@ -123,6 +123,7 @@ </file> <file> <source>src/resources/runbit</source> + <fileMode>0755</fileMode> <outputDirectory>bin</outputDirectory> </file> <file> @@ -131,14 +132,17 @@ </file> <file> <source>src/resources/drillbit.sh</source> + <fileMode>0755</fileMode> <outputDirectory>bin</outputDirectory> </file> <file> <source>src/resources/drill-config.sh</source> + <fileMode>0755</fileMode> <outputDirectory>bin</outputDirectory> </file> <file> <source>src/resources/sqlline</source> + <fileMode>0755</fileMode> <outputDirectory>bin</outputDirectory> </file> <file> @@ -147,10 +151,12 @@ </file> <file> <source>src/resources/submit_plan</source> + <fileMode>0755</fileMode> <outputDirectory>bin</outputDirectory> </file> <file> <source>src/resources/drill_dumpcat</source> + <fileMode>0755</fileMode> <outputDirectory>bin</outputDirectory> </file> <file> @@ -163,6 +169,7 @@ </file> <file> <source>src/resources/drill-env.sh</source> + <fileMode>0755</fileMode> <outputDirectory>conf</outputDirectory> </file> <file> http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8289f726/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/StorageResources.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/StorageResources.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/StorageResources.java index 17ea72c..714e711 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/StorageResources.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/StorageResources.java @@ -20,7 +20,8 @@ package org.apache.drill.exec.server.rest; import java.io.IOException; import java.io.StringReader; import java.net.URI; -import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; import java.util.List; import java.util.Map; @@ -37,6 +38,8 @@ import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; import javax.ws.rs.core.UriInfo; +import org.apache.commons.lang3.tuple.ImmutablePair; +import org.apache.commons.lang3.tuple.Pair; import org.apache.drill.common.exceptions.ExecutionSetupException; import org.apache.drill.common.logical.StoragePluginConfig; import org.apache.drill.exec.store.StoragePlugin; @@ -55,6 +58,13 @@ import freemarker.template.SimpleHash; @Path("/storage") public class StorageResources { static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(StorageResources.class); + + static final Comparator<Pair<String, Boolean>> PLUGIN_NAME_COMPARATOR = new Comparator<Pair<String, Boolean>>() { + @Override + public int compare(Pair<String, Boolean> o1, Pair<String, Boolean> o2) { + return o1.getKey().compareTo(o2.getKey()); + } + }; @Inject StoragePluginRegistry storage; @Inject PStoreProvider storeProvider; @@ -62,17 +72,24 @@ public class StorageResources { @GET @Produces(MediaType.TEXT_HTML) - public Viewable getQueries() { - - List<SimpleHash> list = Lists.newArrayList(); + public Viewable listPlugins() { + // build a list and sort by plugin instance name + List<Pair<String, Boolean>> pluginList = Lists.newArrayList(); for (Map.Entry<String, StoragePluginConfig> config : storage.getStore()) { + pluginList.add(ImmutablePair.of(config.getKey(), config.getValue().isEnabled())); + } + Collections.sort(pluginList, PLUGIN_NAME_COMPARATOR); + + // now use the sorted list to build the freemarker model + List<SimpleHash> modelList = Lists.newArrayList(); + for (Pair<String, Boolean> plugin : pluginList) { SimpleHash map = new SimpleHash(); - map.put("name", config.getKey()); - map.put("enabled", config.getValue().isEnabled()); - list.add(map); + map.put("name", plugin.getLeft()); + map.put("enabled", plugin.getRight()); + modelList.add(map); } - return new Viewable("/rest/storage/list.ftl", list); + return new Viewable("/rest/storage/list.ftl", modelList); } @GET http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8289f726/exec/java-exec/src/main/java/org/apache/drill/exec/store/StoragePluginRegistry.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/StoragePluginRegistry.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/StoragePluginRegistry.java index 4e49a1c..ca1cfe8 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/StoragePluginRegistry.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/StoragePluginRegistry.java @@ -29,6 +29,7 @@ import java.util.Map; import java.util.Map.Entry; import java.util.Set; import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.TimeUnit; import net.hydromatic.optiq.SchemaPlus; import net.hydromatic.optiq.tools.RuleSet; @@ -56,15 +57,21 @@ import org.apache.drill.exec.store.sys.SystemTablePluginConfig; import org.eigenbase.relopt.RelOptRule; import com.google.common.base.Charsets; +import com.google.common.base.Stopwatch; import com.google.common.collect.ImmutableSet; import com.google.common.collect.ImmutableSet.Builder; import com.google.common.collect.Lists; import com.google.common.collect.Maps; +import com.google.common.collect.Sets; import com.google.common.io.Resources; public class StoragePluginRegistry implements Iterable<Map.Entry<String, StoragePlugin>> { static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(StoragePluginRegistry.class); + public static final String SYS_PLUGIN = "sys"; + + public static final String INFORMATION_SCHEMA_PLUGIN = "INFORMATION_SCHEMA"; + private Map<Object, Constructor<? extends StoragePlugin>> availablePlugins = new HashMap<Object, Constructor<? extends StoragePlugin>>(); private ConcurrentMap<String, StoragePlugin> plugins; @@ -175,8 +182,8 @@ public class StoragePluginRegistry implements Iterable<Map.Entry<String, Storage throw new IllegalStateException(e); } - activePlugins.put("INFORMATION_SCHEMA", new InfoSchemaStoragePlugin(new InfoSchemaConfig(), context, "INFORMATION_SCHEMA")); - activePlugins.put("sys", new SystemTablePlugin(SystemTablePluginConfig.INSTANCE, context, "sys")); + activePlugins.put(INFORMATION_SCHEMA_PLUGIN, new InfoSchemaStoragePlugin(new InfoSchemaConfig(), context, INFORMATION_SCHEMA_PLUGIN)); + activePlugins.put(SYS_PLUGIN, new SystemTablePlugin(SystemTablePluginConfig.INSTANCE, context, SYS_PLUGIN)); return activePlugins; } @@ -212,12 +219,10 @@ public class StoragePluginRegistry implements Iterable<Map.Entry<String, Storage public StoragePlugin getPlugin(String name) throws ExecutionSetupException { StoragePlugin plugin = plugins.get(name); - if(name.equals("sys") || name.equals("INFORMATION_SCHEMA")) return plugin; - - StoragePluginConfig config = this.pluginSystemTable.get(name); + if(name.equals(SYS_PLUGIN) || name.equals(INFORMATION_SCHEMA_PLUGIN)) return plugin; // since we lazily manage the list of plugins per server, we need to update this once we know that it is time. - + StoragePluginConfig config = this.pluginSystemTable.get(name); if (config == null) { if(plugin != null) plugins.remove(name); return null; @@ -227,7 +232,6 @@ public class StoragePluginRegistry implements Iterable<Map.Entry<String, Storage } return plugin; } - } public StoragePlugin getPlugin(StoragePluginConfig config) throws ExecutionSetupException { @@ -249,9 +253,10 @@ public class StoragePluginRegistry implements Iterable<Map.Entry<String, Storage private StoragePlugin create(String name, StoragePluginConfig pluginConfig) throws ExecutionSetupException { StoragePlugin plugin = null; Constructor<? extends StoragePlugin> c = availablePlugins.get(pluginConfig.getClass()); - if (c == null) + if (c == null) { throw new ExecutionSetupException(String.format("Failure finding StoragePlugin constructor for config %s", pluginConfig)); + } try { plugin = c.newInstance(pluginConfig, context, name); return plugin; @@ -281,14 +286,29 @@ public class StoragePluginRegistry implements Iterable<Map.Entry<String, Storage @Override public void registerSchemas(UserSession session, SchemaPlus parent) { + Stopwatch watch = new Stopwatch(); + watch.start(); + try { + Set<String> currentPluginNames = Sets.newHashSet(plugins.keySet()); + // iterate through the plugin instances in the persistence store adding + // any new ones and refreshing those whose configuration has changed + for (Map.Entry<String, StoragePluginConfig> config : pluginSystemTable) { + if (config.getValue().isEnabled()) { + getPlugin(config.getKey()); + currentPluginNames.remove(config.getKey()); + } + } + // remove those which are no longer in the registry + for (String pluginName : currentPluginNames) { + if(pluginName.equals(SYS_PLUGIN) || pluginName.equals(INFORMATION_SCHEMA_PLUGIN)) continue; + plugins.remove(pluginName); + } + + // finally register schemas with the refreshed plugins for (StoragePlugin plugin : plugins.values()) { plugin.registerSchemas(session, parent); } - - getPlugin("sys").registerSchemas(session, parent); - getPlugin("INFORMATION_SCHEMA").registerSchemas(session, parent); - } catch (ExecutionSetupException e) { throw new DrillRuntimeException("Failure while updating storage plugins", e); } @@ -333,10 +353,11 @@ public class StoragePluginRegistry implements Iterable<Map.Entry<String, Storage } catch (ClassCastException e) { throw new RuntimeException(String.format("Schema '%s' is not expected under root schema", schema.getName())); } - SubSchemaWrapper wrapper = new SubSchemaWrapper(drillSchema); parent.add(wrapper.getName(), wrapper); } + + logger.debug("Took {} ms to register schemas.", watch.elapsed(TimeUnit.MILLISECONDS)); } } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8289f726/exec/java-exec/src/main/resources/rest/storage/list.ftl ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/resources/rest/storage/list.ftl b/exec/java-exec/src/main/resources/rest/storage/list.ftl index 3636fbb..226bd2b 100644 --- a/exec/java-exec/src/main/resources/rest/storage/list.ftl +++ b/exec/java-exec/src/main/resources/rest/storage/list.ftl @@ -21,20 +21,33 @@ <div class="table-responsive"> <table class="table"> <tbody> + <tr><th colspan="2" style="border:none;">Enabled Plugins</th></tr> <#list model as plugin> - <tr> - <td style="border:none;"> - ${plugin.name} - </td> - <td style="border:none;"> - <a class="btn btn-primary" href="/storage/${plugin.name}/config/update">Update</a> - <#if plugin.enabled> + <#if plugin.enabled == true> + <tr> + <td style="border:none;"> + ${plugin.name} + </td> + <td style="border:none;"> + <a class="btn btn-default" href="/storage/${plugin.name}/config/update">Update</a> <a class="btn btn-default" href="/storage/${plugin.name}/config/enable/false">Disable</a> - <#else> - <a class="btn btn-primary" href="/storage/${plugin.name}/config/enable/true">Enable</a> - </#if> - </td> - </tr> + </td> + </tr> + </#if> + </#list> + <tr><th colspan="2" style="border:none;">Disabled Plugins</th></tr> + <#list model as plugin> + <#if plugin.enabled == false> + <tr> + <td style="border:none;"> + ${plugin.name} + </td> + <td style="border:none;"> + <a class="btn btn-default" href="/storage/${plugin.name}/config/update">Update</a> + <a class="btn btn-default" href="/storage/${plugin.name}/config/enable/true">Enable</a> + </td> + </tr> + </#if> </#list> </tbody> </table> http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8289f726/exec/java-exec/src/main/resources/rest/storage/update.ftl ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/resources/rest/storage/update.ftl b/exec/java-exec/src/main/resources/rest/storage/update.ftl index f4b9e12..3ba375d 100644 --- a/exec/java-exec/src/main/resources/rest/storage/update.ftl +++ b/exec/java-exec/src/main/resources/rest/storage/update.ftl @@ -30,7 +30,7 @@ <#if model.enabled> <a class="btn btn-default" href="/storage/${model.name}/config/enable/false">Disable</a> <#else> - <a class="btn btn-primary" href="/storage/${model.name}/config/enable/true">Enable</a> + <a class="btn btn-default" href="/storage/${model.name}/config/enable/true">Enable</a> </#if> <#if model.exists> <a class="btn btn-danger" href="/storage/${model.name}/config/delete">Delete</a>
