[ https://issues.apache.org/jira/browse/DRILL-5089?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16257633#comment-16257633 ]
ASF GitHub Bot commented on DRILL-5089: --------------------------------------- Github user chunhui-shi commented on a diff in the pull request: https://github.com/apache/drill/pull/1032#discussion_r151798147 --- Diff: exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DynamicRootSchema.java --- @@ -0,0 +1,140 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.planner.sql; + +import com.google.common.collect.ImmutableSortedSet; +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; +import org.apache.calcite.DataContext; +import org.apache.calcite.jdbc.CalciteRootSchema; +import org.apache.calcite.jdbc.CalciteSchema; + +import org.apache.calcite.linq4j.tree.Expression; +import org.apache.calcite.linq4j.tree.Expressions; +import org.apache.calcite.schema.SchemaPlus; +import org.apache.calcite.schema.impl.AbstractSchema; +import org.apache.calcite.util.BuiltInMethod; +import org.apache.calcite.util.Compatible; +import org.apache.drill.common.exceptions.ExecutionSetupException; +import org.apache.drill.exec.store.SchemaConfig; +import org.apache.drill.exec.store.StoragePlugin; +import org.apache.drill.exec.store.StoragePluginRegistry; +import org.apache.drill.exec.store.SubSchemaWrapper; + +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.NavigableSet; +import java.util.Set; + +/** + * This class is to allow us loading schemas from storage plugins later when {@link #getSubSchema(String, boolean)} + * is called. + */ +public class DynamicRootSchema extends DynamicSchema + implements CalciteRootSchema { + private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DynamicRootSchema.class); + /** Creates a root schema. */ + DynamicRootSchema(StoragePluginRegistry storages, SchemaConfig schemaConfig) { + super(null, new RootSchema(), ""); + this.schemaConfig = schemaConfig; + this.storages = storages; + } + + @Override + public CalciteSchema getSubSchema(String schemaName, boolean caseSensitive) { + CalciteSchema retSchema = getSubSchemaMap().get(schemaName); + if (retSchema != null) { + return retSchema; + } + + loadSchemaFactory(schemaName, caseSensitive); + retSchema = getSubSchemaMap().get(schemaName); + return retSchema; + } + + @Override + public NavigableSet<String> getTableNames() { + Set<String> pluginNames = Sets.newHashSet(); + for (Map.Entry<String, StoragePlugin> storageEntry : getSchemaFactories()) { + pluginNames.add(storageEntry.getKey()); + } + return Compatible.INSTANCE.navigableSet( + ImmutableSortedSet.copyOf( + Sets.union(pluginNames, getSubSchemaMap().keySet()))); + } + + /** + * load schema factory(storage plugin) for schemaName + * @param schemaName + * @param caseSensitive + */ + public void loadSchemaFactory(String schemaName, boolean caseSensitive) { + try { + SchemaPlus thisPlus = this.plus(); + StoragePlugin plugin = getSchemaFactories().getPlugin(schemaName); + if (plugin != null) { + plugin.registerSchemas(schemaConfig, thisPlus); + return; + } + + // we could not find the plugin, the schemaName could be `dfs.tmp`, a 2nd level schema under 'dfs' + String[] paths = schemaName.split("\\."); + if (paths.length == 2) { + plugin = getSchemaFactories().getPlugin(paths[0]); + if (plugin == null) { + return; + } + + // we could find storage plugin for first part(e.g. 'dfs') of schemaName (e.g. 'dfs.tmp') + // register schema for this storage plugin to 'this'. + plugin.registerSchemas(schemaConfig, thisPlus); + + // we load second level schemas for this storage plugin + final SchemaPlus firstlevelSchema = thisPlus.getSubSchema(paths[0]); + final List<SchemaPlus> secondLevelSchemas = Lists.newArrayList(); + for (String secondLevelSchemaName : firstlevelSchema.getSubSchemaNames()) { + secondLevelSchemas.add(firstlevelSchema.getSubSchema(secondLevelSchemaName)); + } + + for (SchemaPlus schema : secondLevelSchemas) { + org.apache.drill.exec.store.AbstractSchema drillSchema; + try { + drillSchema = schema.unwrap(org.apache.drill.exec.store.AbstractSchema.class); + } catch (ClassCastException e) { + throw new RuntimeException(String.format("Schema '%s' is not expected under root schema", schema.getName())); + } + SubSchemaWrapper wrapper = new SubSchemaWrapper(drillSchema); + thisPlus.add(wrapper.getName(), wrapper); + } + } + } catch(ExecutionSetupException | IOException ex) { + logger.warn("Failed to load schema for \"" + schemaName + "\"!", ex); + } + } + + static class RootSchema extends AbstractSchema { + @Override public Expression getExpression(SchemaPlus parentSchema, --- End diff -- This is copied from the RootSchema used in SimpleCalciteSchema which class is not public. getExpression is used in Calcite code not in our code. > Skip initializing all enabled storage plugins for every query > ------------------------------------------------------------- > > Key: DRILL-5089 > URL: https://issues.apache.org/jira/browse/DRILL-5089 > Project: Apache Drill > Issue Type: Improvement > Components: Query Planning & Optimization > Affects Versions: 1.9.0 > Reporter: Abhishek Girish > Assignee: Chunhui Shi > Priority: Critical > > In a query's lifecycle, at attempt is made to initialize each enabled storage > plugin, while building the schema tree. This is done regardless of the actual > plugins involved within a query. > Sometimes, when one or more of the enabled storage plugins have issues - > either due to misconfiguration or the underlying datasource being slow or > being down, the overall query time taken increases drastically. Most likely > due the attempt being made to register schemas from a faulty plugin. > For example, when a jdbc plugin is configured with SQL Server, and at one > point the underlying SQL Server db goes down, any Drill query starting to > execute at that point and beyond begin to slow down drastically. > We must skip registering unrelated schemas (& workspaces) for a query. -- This message was sent by Atlassian JIRA (v6.4.14#64029)