LENS-1359: Add driver hooks for user based filtering of queries
Project: http://git-wip-us.apache.org/repos/asf/lens/repo Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/02cbd6bc Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/02cbd6bc Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/02cbd6bc Branch: refs/heads/master Commit: 02cbd6bc3c5663a4bc97e14efd3ab64107f3d9c0 Parents: 078555c Author: Rajat Khandelwal <[email protected]> Authored: Mon Nov 14 11:07:56 2016 +0530 Committer: Puneet <[email protected]> Committed: Mon Nov 14 11:07:56 2016 +0530 ---------------------------------------------------------------------- .../java/org/apache/lens/api/parse/Parser.java | 24 ++++ .../src/main/resources/hivedriver-default.xml | 9 +- .../apache/lens/driver/hive/TestHiveDriver.java | 3 +- .../lens/driver/hive/TestRemoteHiveDriver.java | 4 +- .../lens/server/api/LensConfConstants.java | 9 +- .../server/api/driver/AbstractLensDriver.java | 12 +- .../server/api/driver/DriverConfiguration.java | 28 ++-- .../lens/server/api/driver/DriverQueryHook.java | 101 ------------- .../lens/server/api/driver/LensDriver.java | 1 + .../server/api/driver/NoOpDriverQueryHook.java | 79 ----------- .../driver/hooks/ChainedDriverQueryHook.java | 130 +++++++++++++++++ .../api/driver/hooks/DriverQueryHook.java | 102 +++++++++++++ .../api/driver/hooks/NoOpDriverQueryHook.java | 80 +++++++++++ .../driver/hooks/QueryCostBasedQueryHook.java | 142 +++++++++++++++++++ .../api/driver/hooks/UserBasedQueryHook.java | 71 ++++++++++ .../query/cost/FactPartitionBasedQueryCost.java | 8 ++ .../lens/server/api/query/cost/QueryCost.java | 1 + .../api/driver/DriverConfigurationTest.java | 58 ++++++++ .../hooks/ChainedDriverQueryHookTest.java | 82 +++++++++++ .../hooks/QueryCostBasedQueryHookTest.java | 108 ++++++++++++++ .../driver/hooks/UserBasedQueryHookTest.java | 87 ++++++++++++ .../server/api/user/MockDriverQueryHook.java | 4 +- .../src/main/resources/lensserver-default.xml | 5 + .../drivers/hive/hive1/hivedriver-site.xml | 2 +- .../drivers/hive/hive2/hivedriver-site.xml | 2 +- pom.xml | 2 +- src/site/apt/admin/config.apt | 126 ++++++++-------- src/site/apt/admin/hivedriver-config.apt | 2 +- 28 files changed, 1006 insertions(+), 276 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/lens/blob/02cbd6bc/lens-api/src/main/java/org/apache/lens/api/parse/Parser.java ---------------------------------------------------------------------- diff --git a/lens-api/src/main/java/org/apache/lens/api/parse/Parser.java b/lens-api/src/main/java/org/apache/lens/api/parse/Parser.java new file mode 100644 index 0000000..afa9476 --- /dev/null +++ b/lens-api/src/main/java/org/apache/lens/api/parse/Parser.java @@ -0,0 +1,24 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.lens.api.parse; + + +public interface Parser<T> { + T parse(String value); +} http://git-wip-us.apache.org/repos/asf/lens/blob/02cbd6bc/lens-driver-hive/src/main/resources/hivedriver-default.xml ---------------------------------------------------------------------- diff --git a/lens-driver-hive/src/main/resources/hivedriver-default.xml b/lens-driver-hive/src/main/resources/hivedriver-default.xml index f5fd3bb..a13d3b0 100644 --- a/lens-driver-hive/src/main/resources/hivedriver-default.xml +++ b/lens-driver-hive/src/main/resources/hivedriver-default.xml @@ -35,10 +35,11 @@ </property> <property> - <name>lens.driver.hive.query.hook.class</name> - <value>org.apache.lens.server.api.driver.NoOpDriverQueryHook</value> - <description>The query hook class for hive driver. By default hook is No op. To add a hook, you should look - at the default implementation and from there it'll be easy to derive what value can be added through a new hook + <name>lens.driver.hive.query.hook.classes</name> + <value></value> + <description>The query hook classes for hive driver. By default there are no hooks. To add a hook, you should look + at the default implementation and from there it'll be easy to derive what value can be added through a new hook. + Multiple hooks can be provided by providing comma seperated name of classes. </description> </property> http://git-wip-us.apache.org/repos/asf/lens/blob/02cbd6bc/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java ---------------------------------------------------------------------- diff --git a/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java b/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java index 43b33f3..1261409 100644 --- a/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java +++ b/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java @@ -32,6 +32,7 @@ import org.apache.lens.cube.metadata.UpdatePeriod; import org.apache.lens.server.api.LensConfConstants; import org.apache.lens.server.api.driver.*; import org.apache.lens.server.api.driver.DriverQueryStatus.DriverQueryState; +import org.apache.lens.server.api.driver.hooks.DriverQueryHook; import org.apache.lens.server.api.error.LensException; import org.apache.lens.server.api.query.ExplainQueryContext; import org.apache.lens.server.api.query.PreparedQueryContext; @@ -118,7 +119,7 @@ public class TestHiveDriver { protected void createDriver() throws LensException { driverConf.addResource("drivers/hive/hive1/hivedriver-site.xml"); driverConf.setClass(HiveDriver.HIVE_CONNECTION_CLASS, EmbeddedThriftConnection.class, ThriftConnection.class); - driverConf.setClass(LensConfConstants.DRIVER_HOOK_CLASS_SFX, MockDriverQueryHook.class, DriverQueryHook.class); + driverConf.setClass(LensConfConstants.DRIVER_HOOK_CLASSES_SFX, MockDriverQueryHook.class, DriverQueryHook.class); driverConf.set("hive.lock.manager", "org.apache.hadoop.hive.ql.lockmgr.EmbeddedLockManager"); driverConf.setBoolean(HiveDriver.HS2_CALCULATE_PRIORITY, true); driver = new HiveDriver(); http://git-wip-us.apache.org/repos/asf/lens/blob/02cbd6bc/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestRemoteHiveDriver.java ---------------------------------------------------------------------- diff --git a/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestRemoteHiveDriver.java b/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestRemoteHiveDriver.java index 961ec4e..cc9fff0 100644 --- a/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestRemoteHiveDriver.java +++ b/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestRemoteHiveDriver.java @@ -29,10 +29,10 @@ import java.util.concurrent.atomic.AtomicInteger; import org.apache.lens.api.query.QueryHandle; import org.apache.lens.server.api.LensConfConstants; -import org.apache.lens.server.api.driver.DriverQueryHook; import org.apache.lens.server.api.driver.DriverQueryPlan; import org.apache.lens.server.api.driver.DriverQueryStatus.DriverQueryState; import org.apache.lens.server.api.driver.LensDriver; +import org.apache.lens.server.api.driver.hooks.DriverQueryHook; import org.apache.lens.server.api.error.LensException; import org.apache.lens.server.api.query.QueryContext; import org.apache.lens.server.api.user.MockDriverQueryHook; @@ -148,7 +148,7 @@ public class TestRemoteHiveDriver extends TestHiveDriver { driverConf.addResource("drivers/hive/hive1/hivedriver-site.xml"); driver = new HiveDriver(); driverConf.setBoolean(HiveDriver.HS2_CALCULATE_PRIORITY, true); - driverConf.setClass(LensConfConstants.DRIVER_HOOK_CLASS_SFX, MockDriverQueryHook.class, DriverQueryHook.class); + driverConf.setClass(LensConfConstants.DRIVER_HOOK_CLASSES_SFX, MockDriverQueryHook.class, DriverQueryHook.class); driver.configure(driverConf, "hive", "hive1"); drivers = Lists.<LensDriver>newArrayList(driver); System.out.println("TestRemoteHiveDriver created"); http://git-wip-us.apache.org/repos/asf/lens/blob/02cbd6bc/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java ---------------------------------------------------------------------- diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java b/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java index 7ccb170..7fd487c 100644 --- a/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java +++ b/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java @@ -20,8 +20,11 @@ package org.apache.lens.server.api; import javax.ws.rs.core.MediaType; +import org.apache.lens.api.parse.Parser; import org.apache.lens.server.api.error.LensException; import org.apache.lens.server.api.metastore.*; +import org.apache.lens.server.api.query.cost.FactPartitionBasedQueryCost; +import org.apache.lens.server.api.query.cost.QueryCost; /** * The Class LensConfConstants. @@ -967,7 +970,7 @@ public final class LensConfConstants { /** * Driver hook property */ - public static final String DRIVER_HOOK_CLASS_SFX = "query.hook.class"; + public static final String DRIVER_HOOK_CLASSES_SFX = "query.hook.classes"; /** * Default driver weight @@ -1220,6 +1223,10 @@ public final class LensConfConstants { */ public static final int DEFAULT_MAX_SCHEDULED_JOB_PER_USER = -1; + public static final String QUERY_COST_PARSER = SERVER_PFX + "query.cost.parser.class"; + public static final Class<? extends Parser<? extends QueryCost>> DEFAULT_QUERY_COST_PARSER + = FactPartitionBasedQueryCost.Parser.class; + /** * Maximum number of scheduled job per user. */ http://git-wip-us.apache.org/repos/asf/lens/blob/02cbd6bc/lens-server-api/src/main/java/org/apache/lens/server/api/driver/AbstractLensDriver.java ---------------------------------------------------------------------- diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/AbstractLensDriver.java b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/AbstractLensDriver.java index 8f30aa0..91972a9 100644 --- a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/AbstractLensDriver.java +++ b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/AbstractLensDriver.java @@ -23,6 +23,8 @@ import static org.apache.lens.server.api.util.LensUtil.getImplementations; import org.apache.lens.api.Priority; import org.apache.lens.server.api.LensConfConstants; +import org.apache.lens.server.api.driver.hooks.ChainedDriverQueryHook; +import org.apache.lens.server.api.driver.hooks.DriverQueryHook; import org.apache.lens.server.api.error.LensException; import org.apache.lens.server.api.query.AbstractQueryContext; import org.apache.lens.server.api.query.QueryContext; @@ -85,14 +87,8 @@ public abstract class AbstractLensDriver implements LensDriver { } protected void loadQueryHook() throws LensException { - try { - queryHook = getConf().getClass( - DRIVER_HOOK_CLASS_SFX, NoOpDriverQueryHook.class, DriverQueryHook.class - ).newInstance(); - queryHook.setDriver(this); - } catch (InstantiationException | IllegalAccessException e) { - throw new LensException("Can't instantiate driver query hook for hivedriver with given class", e); - } + this.queryHook = ChainedDriverQueryHook.from(getConf(), DRIVER_HOOK_CLASSES_SFX); + queryHook.setDriver(this); } protected void loadRetryPolicyDecider() throws LensException { http://git-wip-us.apache.org/repos/asf/lens/blob/02cbd6bc/lens-server-api/src/main/java/org/apache/lens/server/api/driver/DriverConfiguration.java ---------------------------------------------------------------------- diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/DriverConfiguration.java b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/DriverConfiguration.java index 69a1a0b..f072c73 100644 --- a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/DriverConfiguration.java +++ b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/DriverConfiguration.java @@ -34,27 +34,31 @@ public class DriverConfiguration extends Configuration { this.driverClass = driverClass; this.driverClassType = driverClass.getSimpleName().toLowerCase().replaceAll("driver$", ""); } + public DriverConfiguration(String driverType, Class<? extends AbstractLensDriver> driverClass) { + this.driverType = driverType; + this.driverClass = driverClass; + this.driverClassType = driverClass.getSimpleName().toLowerCase().replaceAll("driver$", ""); + } @Override - public String[] getStrings(String name) { - for (String key : new String[]{DRIVER_PFX + driverType + "." + name, DRIVER_PFX + driverClassType + "." + name, - DRIVER_PFX + name, name, }) { - String[] s = super.getStrings(key); - if (s != null) { - return s; + public String get(String name) { + String[] prefixes = new String[]{DRIVER_PFX + driverType + ".", DRIVER_PFX + driverClassType + ".", DRIVER_PFX, }; + for (String prefix : prefixes) { + if (name.startsWith(prefix)) { + return getInternal(name.substring(prefix.length())); } } - return null; + return getInternal(name); } - @Override - public <U> Class<? extends U> getClass(String name, Class<? extends U> defaultValue, Class<U> xface) { + public String getInternal(String name) { for (String key : new String[]{DRIVER_PFX + driverType + "." + name, DRIVER_PFX + driverClassType + "." + name, DRIVER_PFX + name, name, }) { - if (getTrimmed(key) != null) { - return super.getClass(key, defaultValue, xface); + String val = super.get(key); + if (val != null) { + return val; } } - return defaultValue; + return null; } } http://git-wip-us.apache.org/repos/asf/lens/blob/02cbd6bc/lens-server-api/src/main/java/org/apache/lens/server/api/driver/DriverQueryHook.java ---------------------------------------------------------------------- diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/DriverQueryHook.java b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/DriverQueryHook.java deleted file mode 100644 index f8a9ee0..0000000 --- a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/DriverQueryHook.java +++ /dev/null @@ -1,101 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -/* - * - */ -package org.apache.lens.server.api.driver; - -import org.apache.lens.server.api.error.LensException; -import org.apache.lens.server.api.query.AbstractQueryContext; -import org.apache.lens.server.api.query.QueryContext; - -/** - * Drivers should initialize a DriverQueryHook object in their initialization and expose it - * via {@link LensDriver#getQueryHook}. Lens Server will invoke the driver hook at relevant points during - * query execution. By default each driver exposes a {@link NoOpDriverQueryHook} which does nothing when invoked. - * - * The only use case I see right now is to provide a hook just after driver has been selected for a query and - * before query is launched on the driver. One example usage for hive driver would be to add dynamic configuration or - * stall execution of a query by looking at the final translated query itself (based on table involved, filters - * involved, etc in the query). - * - * This interface is expected to evolve for some time as more needs for hook are discovered - * - * Note: Note if the hook updates any configuration, same should be reflected in QueryContext - * via {@link AbstractQueryContext#updateConf(Map)} to ensure the modified configuration is persisted and is available - * on server restarts and other bookkeeping needs. - */ -public interface DriverQueryHook { - - /** - * This setter method is called by the driver once hook instance is created. This driver information can be used while - * extracting driver specific information form the QueryContext. - * @param driver - */ - void setDriver(LensDriver driver); - - /** - * Called just before rewrite operation is tried on this driver - * - * @param ctx - * @throws LensException - */ - void preRewrite(AbstractQueryContext ctx) throws LensException; - - /** - * Called just after a successful rewrite operation is tried on this driver - * - * @param ctx - * @throws LensException - */ - void postRewrite(AbstractQueryContext ctx) throws LensException; - - /** - * Called just before estimate operation is tried on this driver - * Note : Estimate operation will be skipped if rewrite operation fails for this driver - * - * @param ctx - * @throws LensException - */ - void preEstimate(AbstractQueryContext ctx) throws LensException; - - /** - * Called just after a successful estimate operation is tried on this driver - * - * @param ctx - * @throws LensException - */ - void postEstimate(AbstractQueryContext ctx) throws LensException; - - /** - * Called just after driver has been selected to execute a query. - * - * @param ctx - * @throws LensException - */ - void postDriverSelection(AbstractQueryContext ctx) throws LensException; - - /** - * Called just before launching the query on the selected driver. - * @param ctx - * @throws LensException - */ - void preLaunch(QueryContext ctx) throws LensException; - -} http://git-wip-us.apache.org/repos/asf/lens/blob/02cbd6bc/lens-server-api/src/main/java/org/apache/lens/server/api/driver/LensDriver.java ---------------------------------------------------------------------- diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/LensDriver.java b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/LensDriver.java index 1462239..0080b97 100644 --- a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/LensDriver.java +++ b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/LensDriver.java @@ -23,6 +23,7 @@ import java.io.Externalizable; import org.apache.lens.api.Priority; import org.apache.lens.api.query.QueryHandle; import org.apache.lens.api.query.QueryPrepareHandle; +import org.apache.lens.server.api.driver.hooks.DriverQueryHook; import org.apache.lens.server.api.error.LensException; import org.apache.lens.server.api.events.LensEventListener; import org.apache.lens.server.api.query.AbstractQueryContext; http://git-wip-us.apache.org/repos/asf/lens/blob/02cbd6bc/lens-server-api/src/main/java/org/apache/lens/server/api/driver/NoOpDriverQueryHook.java ---------------------------------------------------------------------- diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/NoOpDriverQueryHook.java b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/NoOpDriverQueryHook.java deleted file mode 100644 index 4f1f2eb..0000000 --- a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/NoOpDriverQueryHook.java +++ /dev/null @@ -1,79 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -/* - * - */ -package org.apache.lens.server.api.driver; - -import org.apache.lens.server.api.error.LensException; -import org.apache.lens.server.api.query.AbstractQueryContext; -import org.apache.lens.server.api.query.QueryContext; - -import lombok.Getter; -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class NoOpDriverQueryHook implements DriverQueryHook { - - @Getter - private LensDriver driver; - - @Override - public void setDriver(LensDriver driver) { - this.driver = driver; - log.debug("The Driver for this driver query hook is {}", driver.getFullyQualifiedName()); - } - - @Override - public void preRewrite(AbstractQueryContext ctx) throws LensException { - log.debug("Pre rewrite for user {}, user query: {}, driver: {}", ctx.getSubmittedUser(), ctx.getUserQuery(), - driver.getFullyQualifiedName()); - } - - @Override - public void postRewrite(AbstractQueryContext ctx) throws LensException { - log.debug("Post rewrite for user {}, user query: {}, driver: {}, driver query :{}", ctx.getSubmittedUser(), - ctx.getUserQuery(), driver.getFullyQualifiedName(), ctx.getDriverQuery(driver)); - } - - @Override - public void preEstimate(AbstractQueryContext ctx) throws LensException { - log.debug("Pre estimate for user {}, user query: {}, driver: {}, driver query :{}", ctx.getSubmittedUser(), - ctx.getUserQuery(), driver.getFullyQualifiedName(), ctx.getDriverQuery(driver)); - } - - @Override - public void postEstimate(AbstractQueryContext ctx) throws LensException { - log.debug("Post estimate for user {}, user query: {}, driver: {}, driver query :{}, query cost :{}", - ctx.getSubmittedUser(), ctx.getUserQuery(), driver.getFullyQualifiedName(), ctx.getDriverQuery(driver), - ctx.getDriverQueryCost(driver)); - } - - @Override - public void postDriverSelection(AbstractQueryContext ctx) throws LensException { - log.debug("Post driver selection for user {}, user query: {}, driver {}, driver query: {}", ctx.getSubmittedUser(), - ctx.getUserQuery(), ctx.getSelectedDriver().getFullyQualifiedName(), ctx.getSelectedDriverQuery()); - } - - @Override - public void preLaunch(QueryContext ctx) { - log.debug("Pre launch for user {}, user query: {}, driver {}, driver query: {}", ctx.getSubmittedUser(), - ctx.getUserQuery(), ctx.getSelectedDriver().getFullyQualifiedName(), ctx.getSelectedDriverQuery()); - } -} http://git-wip-us.apache.org/repos/asf/lens/blob/02cbd6bc/lens-server-api/src/main/java/org/apache/lens/server/api/driver/hooks/ChainedDriverQueryHook.java ---------------------------------------------------------------------- diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/hooks/ChainedDriverQueryHook.java b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/hooks/ChainedDriverQueryHook.java new file mode 100644 index 0000000..564028c --- /dev/null +++ b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/hooks/ChainedDriverQueryHook.java @@ -0,0 +1,130 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +/* + * + */ +package org.apache.lens.server.api.driver.hooks; + +import java.util.List; + +import org.apache.lens.server.api.driver.LensDriver; +import org.apache.lens.server.api.error.LensException; +import org.apache.lens.server.api.query.AbstractQueryContext; +import org.apache.lens.server.api.query.QueryContext; + +import org.apache.hadoop.conf.Configurable; +import org.apache.hadoop.conf.Configuration; + +import com.google.common.collect.Lists; +import lombok.Data; + +/** + * This hook allows chaining of different hooks. A driver can specify multiple hooks for itself by providing + * `query.hook.classes` in its configuration. The value should be a comma separated list of class names, each + * of which should be an implementation of org.apache.lens.server.api.driver.hooks.DriverQueryHook. This hook + * stores instances of those classes. + * All the methods of this class invokes the same method on each of the hooks in the chain. + */ +@Data +public class ChainedDriverQueryHook extends NoOpDriverQueryHook { + + private final Iterable<DriverQueryHook> hooks; + + @Override + public void setDriver(LensDriver driver) { + super.setDriver(driver); + for (DriverQueryHook hook : hooks) { + hook.setDriver(driver); + } + } + + @Override + public void preRewrite(AbstractQueryContext ctx) throws LensException { + super.preRewrite(ctx); + for (DriverQueryHook hook : hooks) { + hook.preRewrite(ctx); + } + } + + @Override + public void postRewrite(AbstractQueryContext ctx) throws LensException { + super.postRewrite(ctx); + for (DriverQueryHook hook : hooks) { + hook.postRewrite(ctx); + } + } + + @Override + public void preEstimate(AbstractQueryContext ctx) throws LensException { + super.preEstimate(ctx); + for (DriverQueryHook hook : hooks) { + hook.preEstimate(ctx); + } + } + + @Override + public void postEstimate(AbstractQueryContext ctx) throws LensException { + super.postEstimate(ctx); + for (DriverQueryHook hook : hooks) { + hook.postEstimate(ctx); + } + } + + @Override + public void postDriverSelection(AbstractQueryContext ctx) throws LensException { + super.postDriverSelection(ctx); + for (DriverQueryHook hook : hooks) { + hook.postDriverSelection(ctx); + } + } + + @Override + public void preLaunch(QueryContext ctx) throws LensException { + super.preLaunch(ctx); + for (DriverQueryHook hook : hooks) { + hook.preLaunch(ctx); + } + } + + public static ChainedDriverQueryHook from(Configuration conf, String key) throws LensException { + String[] classNames = conf.getStrings(key); + List<DriverQueryHook> hooks = Lists.newArrayList(); + if (classNames != null) { + for (String className : classNames) { + Class<? extends DriverQueryHook> clazz; + try { + clazz = conf.getClassByName(className).asSubclass(DriverQueryHook.class); + } catch (ClassNotFoundException e) { + throw new LensException("Couldn't load class " + className, e); + } + DriverQueryHook instance; + try { + instance = clazz.newInstance(); + } catch (InstantiationException | IllegalAccessException e) { + throw new LensException("Couldn't create instance of class " + clazz.getName(), e); + } + if (instance instanceof Configurable) { + ((Configurable) instance).setConf(conf); + } + hooks.add(instance); + } + } + return new ChainedDriverQueryHook(hooks); + } +} http://git-wip-us.apache.org/repos/asf/lens/blob/02cbd6bc/lens-server-api/src/main/java/org/apache/lens/server/api/driver/hooks/DriverQueryHook.java ---------------------------------------------------------------------- diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/hooks/DriverQueryHook.java b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/hooks/DriverQueryHook.java new file mode 100644 index 0000000..316bb7e --- /dev/null +++ b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/hooks/DriverQueryHook.java @@ -0,0 +1,102 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +/* + * + */ +package org.apache.lens.server.api.driver.hooks; + +import org.apache.lens.server.api.driver.LensDriver; +import org.apache.lens.server.api.error.LensException; +import org.apache.lens.server.api.query.AbstractQueryContext; +import org.apache.lens.server.api.query.QueryContext; + +/** + * Drivers should initialize a DriverQueryHook object in their initialization and expose it + * via {@link LensDriver#getQueryHook}. Lens Server will invoke the driver hook at relevant points during + * query execution. By default each driver exposes a {@link NoOpDriverQueryHook} which does nothing when invoked. + * + * The only use case I see right now is to provide a hook just after driver has been selected for a query and + * before query is launched on the driver. One example usage for hive driver would be to add dynamic configuration or + * stall execution of a query by looking at the final translated query itself (based on table involved, filters + * involved, etc in the query). + * + * This interface is expected to evolve for some time as more needs for hook are discovered + * + * Note: Note if the hook updates any configuration, same should be reflected in QueryContext + * via {@link AbstractQueryContext#updateConf(java.util.Map)} to ensure the modified configuration is persisted and + * is available on server restarts and other bookkeeping needs. + */ +public interface DriverQueryHook { + + /** + * This setter method is called by the driver once hook instance is created. This driver information can be used while + * extracting driver specific information form the QueryContext. + * @param driver + */ + void setDriver(LensDriver driver); + + /** + * Called just before rewrite operation is tried on this driver + * + * @param ctx + * @throws LensException + */ + void preRewrite(AbstractQueryContext ctx) throws LensException; + + /** + * Called just after a successful rewrite operation is tried on this driver + * + * @param ctx + * @throws LensException + */ + void postRewrite(AbstractQueryContext ctx) throws LensException; + + /** + * Called just before estimate operation is tried on this driver + * Note : Estimate operation will be skipped if rewrite operation fails for this driver + * + * @param ctx + * @throws LensException + */ + void preEstimate(AbstractQueryContext ctx) throws LensException; + + /** + * Called just after a successful estimate operation is tried on this driver + * + * @param ctx + * @throws LensException + */ + void postEstimate(AbstractQueryContext ctx) throws LensException; + + /** + * Called just after driver has been selected to execute a query. + * + * @param ctx + * @throws LensException + */ + void postDriverSelection(AbstractQueryContext ctx) throws LensException; + + /** + * Called just before launching the query on the selected driver. + * @param ctx + * @throws LensException + */ + void preLaunch(QueryContext ctx) throws LensException; + +} http://git-wip-us.apache.org/repos/asf/lens/blob/02cbd6bc/lens-server-api/src/main/java/org/apache/lens/server/api/driver/hooks/NoOpDriverQueryHook.java ---------------------------------------------------------------------- diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/hooks/NoOpDriverQueryHook.java b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/hooks/NoOpDriverQueryHook.java new file mode 100644 index 0000000..a6af091 --- /dev/null +++ b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/hooks/NoOpDriverQueryHook.java @@ -0,0 +1,80 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +/* + * + */ +package org.apache.lens.server.api.driver.hooks; + +import org.apache.lens.server.api.driver.LensDriver; +import org.apache.lens.server.api.error.LensException; +import org.apache.lens.server.api.query.AbstractQueryContext; +import org.apache.lens.server.api.query.QueryContext; + +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class NoOpDriverQueryHook implements DriverQueryHook { + + @Getter + private LensDriver driver; + + @Override + public void setDriver(LensDriver driver) { + this.driver = driver; + log.debug("The Driver for this driver query hook is {}", driver.getFullyQualifiedName()); + } + + @Override + public void preRewrite(AbstractQueryContext ctx) throws LensException { + log.debug("Pre rewrite for user {}, user query: {}, driver: {}", ctx.getSubmittedUser(), ctx.getUserQuery(), + driver.getFullyQualifiedName()); + } + + @Override + public void postRewrite(AbstractQueryContext ctx) throws LensException { + log.debug("Post rewrite for user {}, user query: {}, driver: {}, driver query :{}", ctx.getSubmittedUser(), + ctx.getUserQuery(), driver.getFullyQualifiedName(), ctx.getDriverQuery(driver)); + } + + @Override + public void preEstimate(AbstractQueryContext ctx) throws LensException { + log.debug("Pre estimate for user {}, user query: {}, driver: {}, driver query :{}", ctx.getSubmittedUser(), + ctx.getUserQuery(), driver.getFullyQualifiedName(), ctx.getDriverQuery(driver)); + } + + @Override + public void postEstimate(AbstractQueryContext ctx) throws LensException { + log.debug("Post estimate for user {}, user query: {}, driver: {}, driver query :{}, query cost :{}", + ctx.getSubmittedUser(), ctx.getUserQuery(), driver.getFullyQualifiedName(), ctx.getDriverQuery(driver), + ctx.getDriverQueryCost(driver)); + } + + @Override + public void postDriverSelection(AbstractQueryContext ctx) throws LensException { + log.debug("Post driver selection for user {}, user query: {}, driver {}, driver query: {}", ctx.getSubmittedUser(), + ctx.getUserQuery(), ctx.getSelectedDriver().getFullyQualifiedName(), ctx.getSelectedDriverQuery()); + } + + @Override + public void preLaunch(QueryContext ctx) throws LensException { + log.debug("Pre launch for user {}, user query: {}, driver {}, driver query: {}", ctx.getSubmittedUser(), + ctx.getUserQuery(), ctx.getSelectedDriver().getFullyQualifiedName(), ctx.getSelectedDriverQuery()); + } +} http://git-wip-us.apache.org/repos/asf/lens/blob/02cbd6bc/lens-server-api/src/main/java/org/apache/lens/server/api/driver/hooks/QueryCostBasedQueryHook.java ---------------------------------------------------------------------- diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/hooks/QueryCostBasedQueryHook.java b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/hooks/QueryCostBasedQueryHook.java new file mode 100644 index 0000000..8cfe472 --- /dev/null +++ b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/hooks/QueryCostBasedQueryHook.java @@ -0,0 +1,142 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +/* + * + */ +package org.apache.lens.server.api.driver.hooks; + +import static org.apache.lens.server.api.LensConfConstants.DEFAULT_QUERY_COST_PARSER; +import static org.apache.lens.server.api.LensConfConstants.QUERY_COST_PARSER; + +import org.apache.lens.api.parse.Parser; +import org.apache.lens.server.api.driver.LensDriver; +import org.apache.lens.server.api.error.LensException; +import org.apache.lens.server.api.query.AbstractQueryContext; +import org.apache.lens.server.api.query.cost.QueryCost; + +import com.google.common.collect.*; + +/** + * This hook allows for a driver to discard queries based on the cost of the query. This works on two + * configurations values, among which either, both or none can be set. The configuration values are picked from + * driver configuration. The driver config can set `allowed.range.sets` as a mathematical range of costs that + * are allowed on this driver. Queries having cost among these will be allowed + * and the rest will be discarded. On the other hand, the driver can set `disallowed.range.sets` as a + * mathematical range of costs that are disallowed on this driver. Queries having range among these + * will be rejected, and the rest will be allowed. + * + * Mathematical range looks like the following: + * + * (10, 20] U (30, 40) U [1000, ) + * + * + */ +public class QueryCostBasedQueryHook<T extends QueryCost<T>> extends NoOpDriverQueryHook { + public static final String ALLOWED_RANGE_SETS = "allowed.range.sets"; + public static final String DISALLOWED_RANGE_SETS = "disallowed.range.sets"; + private RangeSet<T> allowedCosts; + private Parser<T> parser; + + public RangeSet<T> parseAllowedRangeSet(String rangeStr) { + RangeSet<T> parsed = parseRangeSet(rangeStr); + if (parsed == null) { + TreeRangeSet<T> set = TreeRangeSet.create(); + set.add(Range.<T>all()); + return set; + } else { + return parsed; + } + } + + public RangeSet<T> parseDisallowedRangeSet(String rangeStr) { + RangeSet<T> parsed = parseRangeSet(rangeStr); + if (parsed == null) { + return TreeRangeSet.create(); + } else { + return parsed; + } + } + + public RangeSet<T> parseRangeSet(String rangeStr) { + if (rangeStr == null) { + return null; + } + RangeSet<T> set = TreeRangeSet.create(); + for (String range : rangeStr.split("U")) { + set.add(parseRange(range)); + } + return set; + } + + public Range<T> parseRange(String rangeStr) { + if (rangeStr == null) { + return null; + } + rangeStr = rangeStr.trim(); + BoundType lowerBound, upperBound; + if (rangeStr.startsWith("[")) { + lowerBound = BoundType.CLOSED; + } else if (rangeStr.startsWith("(")) { + lowerBound = BoundType.OPEN; + } else { + throw new IllegalArgumentException("Range should start with either ( or ["); + } + if (rangeStr.endsWith("]")) { + upperBound = BoundType.CLOSED; + } else if (rangeStr.endsWith(")")) { + upperBound = BoundType.OPEN; + } else { + throw new IllegalArgumentException("Range should end with either ) or ]"); + } + String[] pair = rangeStr.substring(1, rangeStr.length() - 1).split(","); + String leftStr = pair[0].trim(); + String rightStr = pair[1].trim(); + if (leftStr.isEmpty() && rightStr.isEmpty()) { + return Range.all(); + } else if (leftStr.isEmpty()) { + return Range.upTo(parser.parse(rightStr), upperBound); + } else if (rightStr.isEmpty()) { + return Range.downTo(parser.parse(leftStr), lowerBound); + } else { + return Range.range(parser.parse(leftStr), lowerBound, parser.parse(rightStr), upperBound); + } + } + + @Override + public void setDriver(LensDriver driver) { + super.setDriver(driver); + Class<? extends Parser<T>> parserClass = (Class<? extends Parser<T>>) driver.getConf().getClass(QUERY_COST_PARSER, + DEFAULT_QUERY_COST_PARSER, Parser.class); + try { + this.parser = parserClass.newInstance(); + } catch (InstantiationException | IllegalAccessException e) { + throw new IllegalArgumentException("Couldn't initialize query cost parser from class " + parserClass); + } + allowedCosts = parseAllowedRangeSet(driver.getConf().get(ALLOWED_RANGE_SETS)); + allowedCosts.removeAll(parseDisallowedRangeSet(driver.getConf().get(DISALLOWED_RANGE_SETS))); + } + + @Override + public void postEstimate(AbstractQueryContext ctx) throws LensException { + if (!allowedCosts.contains((T) ctx.getDriverQueryCost(getDriver()))) { + throw new LensException("Driver " + getDriver() + " doesn't accept query " + + "with cost " + ctx.getSelectedDriverQueryCost() + ". Allowed ranges: " + allowedCosts); + } + } +} http://git-wip-us.apache.org/repos/asf/lens/blob/02cbd6bc/lens-server-api/src/main/java/org/apache/lens/server/api/driver/hooks/UserBasedQueryHook.java ---------------------------------------------------------------------- diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/hooks/UserBasedQueryHook.java b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/hooks/UserBasedQueryHook.java new file mode 100644 index 0000000..1f51db0 --- /dev/null +++ b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/hooks/UserBasedQueryHook.java @@ -0,0 +1,71 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +/* + * + */ +package org.apache.lens.server.api.driver.hooks; + +import java.util.Set; + +import org.apache.lens.server.api.driver.LensDriver; +import org.apache.lens.server.api.error.LensException; +import org.apache.lens.server.api.query.AbstractQueryContext; + +import com.google.common.collect.Sets; + +/** + * This hook allows for a driver to discard queries based on the submitter of the query. This works on two + * configurations values, among which either or none can be set. The configuration values are picked from + * driver configuration. The driver config can set `allowed.users` as a comma separated list of users that + * are allowed to submit queries on this driver. Queries from only those users will be allowed + * and the rest will be discarded. On the other hand, the driver can set `disallowed.users` as a + * comma separated list of users that are disallowed to submit queries on this driver. Queries from these users + * will be rejected, while queries from any other user will be accepted. + */ +public class UserBasedQueryHook extends NoOpDriverQueryHook { + public static final String ALLOWED_USERS = "allowed.users"; + public static final String DISALLOWED_USERS = "disallowed.users"; + private Set<String> allowedUsers; + private Set<String> disallowedUsers; + + @Override + public void setDriver(LensDriver driver) { + super.setDriver(driver); + String[] allowedUsers = driver.getConf().getStrings(ALLOWED_USERS); + String[] disallowedUsers = driver.getConf().getStrings(DISALLOWED_USERS); + if (allowedUsers != null && disallowedUsers != null) { + throw new IllegalStateException("You can't specify both allowed users and disallowed users"); + } + this.allowedUsers = allowedUsers == null ? null : Sets.newHashSet(allowedUsers); + this.disallowedUsers = disallowedUsers == null ? null : Sets.newHashSet(disallowedUsers); + } + + private String getErrorMessage(AbstractQueryContext ctx) { + return "User " + ctx.getSubmittedUser() + " not allowed to submit query on driver " + + getDriver().getFullyQualifiedName(); + } + + @Override + public void preRewrite(AbstractQueryContext ctx) throws LensException { + if ((allowedUsers != null && !allowedUsers.contains(ctx.getSubmittedUser())) + || (disallowedUsers != null && disallowedUsers.contains(ctx.getSubmittedUser()))) { + throw new LensException(getErrorMessage(ctx)); + } + } +} http://git-wip-us.apache.org/repos/asf/lens/blob/02cbd6bc/lens-server-api/src/main/java/org/apache/lens/server/api/query/cost/FactPartitionBasedQueryCost.java ---------------------------------------------------------------------- diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/cost/FactPartitionBasedQueryCost.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/cost/FactPartitionBasedQueryCost.java index 4768550..eba8f0d 100644 --- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/cost/FactPartitionBasedQueryCost.java +++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/cost/FactPartitionBasedQueryCost.java @@ -69,4 +69,12 @@ public class FactPartitionBasedQueryCost implements QueryCost<FactPartitionBased public String toString() { return getQueryCostType() + "(" + getEstimatedResourceUsage() + ")"; } + + public static class Parser implements org.apache.lens.api.parse.Parser<FactPartitionBasedQueryCost> { + + @Override + public FactPartitionBasedQueryCost parse(String value) { + return new FactPartitionBasedQueryCost(Double.parseDouble(value)); + } + } } http://git-wip-us.apache.org/repos/asf/lens/blob/02cbd6bc/lens-server-api/src/main/java/org/apache/lens/server/api/query/cost/QueryCost.java ---------------------------------------------------------------------- diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/cost/QueryCost.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/cost/QueryCost.java index 4712f11..9d7320a 100644 --- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/cost/QueryCost.java +++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/cost/QueryCost.java @@ -37,4 +37,5 @@ public interface QueryCost<T extends QueryCost> extends Comparable<T> { long getEstimatedExecTimeMillis() throws UnsupportedOperationException; double getEstimatedResourceUsage() throws UnsupportedOperationException; + } http://git-wip-us.apache.org/repos/asf/lens/blob/02cbd6bc/lens-server-api/src/test/java/org/apache/lens/server/api/driver/DriverConfigurationTest.java ---------------------------------------------------------------------- diff --git a/lens-server-api/src/test/java/org/apache/lens/server/api/driver/DriverConfigurationTest.java b/lens-server-api/src/test/java/org/apache/lens/server/api/driver/DriverConfigurationTest.java new file mode 100644 index 0000000..2a9cac1 --- /dev/null +++ b/lens-server-api/src/test/java/org/apache/lens/server/api/driver/DriverConfigurationTest.java @@ -0,0 +1,58 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.lens.server.api.driver; + +import static org.apache.lens.server.api.LensConfConstants.DRIVER_PFX; + +import static org.testng.Assert.*; + +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +public class DriverConfigurationTest { + public static final String DRIVER_TYPE = "type"; + public static final String DRIVER_CLASS_TYPE = "mock"; + public static final String KEY = "key"; + public static final String VALUE = "value"; + public static final String[] PREFIXES = new String[]{ + DRIVER_PFX + DRIVER_TYPE + ".", + DRIVER_PFX + DRIVER_CLASS_TYPE + ".", + DRIVER_PFX, + "", + }; + + @DataProvider + public Object[][] keyData() { + Object[][] data = new Object[16][2]; + for (int i = 0; i < 4; i++) { + for (int j = 0; j < 4; j++) { + data[i * 4 + j][0] = PREFIXES[i] + KEY; + data[i * 4 + j][1] = PREFIXES[j] + KEY; + } + } + return data; + } + + @Test(dataProvider = "keyData") + public void testSetAndGet(String keyToSet, String keyToGet) { + DriverConfiguration conf = new DriverConfiguration(DRIVER_TYPE, MockDriver.class); + conf.set(keyToSet, VALUE); + assertEquals(conf.get(keyToGet), VALUE); + } +} http://git-wip-us.apache.org/repos/asf/lens/blob/02cbd6bc/lens-server-api/src/test/java/org/apache/lens/server/api/driver/hooks/ChainedDriverQueryHookTest.java ---------------------------------------------------------------------- diff --git a/lens-server-api/src/test/java/org/apache/lens/server/api/driver/hooks/ChainedDriverQueryHookTest.java b/lens-server-api/src/test/java/org/apache/lens/server/api/driver/hooks/ChainedDriverQueryHookTest.java new file mode 100644 index 0000000..3ea2c42 --- /dev/null +++ b/lens-server-api/src/test/java/org/apache/lens/server/api/driver/hooks/ChainedDriverQueryHookTest.java @@ -0,0 +1,82 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +/* + * + */ +package org.apache.lens.server.api.driver.hooks; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static org.testng.Assert.*; + +import org.apache.lens.server.api.driver.LensDriver; +import org.apache.lens.server.api.error.LensException; +import org.apache.lens.server.api.query.AbstractQueryContext; +import org.apache.lens.server.api.query.cost.FactPartitionBasedQueryCost; + +import org.apache.hadoop.conf.Configuration; + +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + + +public class ChainedDriverQueryHookTest { + private AbstractQueryContext ctx; + ChainedDriverQueryHook hook; + private LensDriver driver; + + @BeforeMethod + public void setUp() throws Exception { + driver = mock(LensDriver.class); + Configuration conf = new Configuration(); + when(driver.getConf()).thenReturn(conf); + ctx = mock(AbstractQueryContext.class); + conf.set(UserBasedQueryHook.ALLOWED_USERS, "user1,user2"); + conf.set(QueryCostBasedQueryHook.ALLOWED_RANGE_SETS, "[0, 10)"); + conf.set("chain", UserBasedQueryHook.class.getName() + "," + QueryCostBasedQueryHook.class.getName()); + hook = ChainedDriverQueryHook.from(conf, "chain"); + hook.setDriver(driver); + } + + @DataProvider + public Object[][] provideData() { + return new Object[][]{ + {"user1", 0.0, true}, // allowed by both + {"user3", 0.0, false}, // disallowed by former + {"user1", 10.0, false}, // disallowed by latter + {"user3", 11.0, false}, // disallowed by both + }; + } + + @Test(dataProvider = "provideData") + public void testChaining(String user, Double cost, boolean allowed) { + when(ctx.getDriverQueryCost(driver)).thenReturn(new FactPartitionBasedQueryCost(cost)); + when(ctx.getSubmittedUser()).thenReturn(user); + try { + hook.preRewrite(ctx); + hook.postRewrite(ctx); + hook.preEstimate(ctx); + hook.postEstimate(ctx); + assertTrue(allowed); + } catch (LensException e) { + assertFalse(allowed); + } + } +} http://git-wip-us.apache.org/repos/asf/lens/blob/02cbd6bc/lens-server-api/src/test/java/org/apache/lens/server/api/driver/hooks/QueryCostBasedQueryHookTest.java ---------------------------------------------------------------------- diff --git a/lens-server-api/src/test/java/org/apache/lens/server/api/driver/hooks/QueryCostBasedQueryHookTest.java b/lens-server-api/src/test/java/org/apache/lens/server/api/driver/hooks/QueryCostBasedQueryHookTest.java new file mode 100644 index 0000000..80cddf3 --- /dev/null +++ b/lens-server-api/src/test/java/org/apache/lens/server/api/driver/hooks/QueryCostBasedQueryHookTest.java @@ -0,0 +1,108 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +/* + * + */ +package org.apache.lens.server.api.driver.hooks; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertTrue; + +import org.apache.lens.server.api.driver.LensDriver; +import org.apache.lens.server.api.query.AbstractQueryContext; +import org.apache.lens.server.api.query.cost.FactPartitionBasedQueryCost; + +import org.apache.hadoop.conf.Configuration; + +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +import com.google.common.collect.RangeSet; + + +public class QueryCostBasedQueryHookTest { + + private QueryCostBasedQueryHook hook = new QueryCostBasedQueryHook(); + private AbstractQueryContext ctx; + private LensDriver driver; + + @BeforeMethod + public void setUp() throws Exception { + driver = mock(LensDriver.class); + Configuration conf = new Configuration(); + conf.set(QueryCostBasedQueryHook.ALLOWED_RANGE_SETS, "[10, 100]"); + conf.set(QueryCostBasedQueryHook.DISALLOWED_RANGE_SETS, "[40, 50]"); + when(driver.getConf()).thenReturn(conf); + ctx = mock(AbstractQueryContext.class); + hook.setDriver(driver); + } + + @DataProvider + public Object[][] provideParsingData() { + return new Object[][]{ + {"(, 5) U [5, 10) U [100, )", new Double[]{0.0, 1.0, 5.0, 100.0}, new Double[]{10.0, 99.0}}, + {"(, )", new Double[]{0.0, 1.0, 10.0, 100.0, 1000000.0}, new Double[]{}}, + {"(5, 10)", new Double[]{6.0, 7.0, 9.9}, new Double[]{2.0, 5.0, 10.0, 20.0}}, + {"[5, 10]", new Double[]{5.0, 6.0, 7.0, 9.9, 10.0}, new Double[]{2.0, 4.9, 10.1, 20.0}}, + }; + } + + @Test(dataProvider = "provideParsingData") + public void testParse(String rangeString, Double[] includes, Double[] excludes) { + RangeSet<FactPartitionBasedQueryCost> range = hook.parseRangeSet(rangeString); + for (Double cost : includes) { + assertTrue(range.contains(new FactPartitionBasedQueryCost(cost))); + } + for (Double cost : excludes) { + assertFalse(range.contains(new FactPartitionBasedQueryCost(cost))); + } + } + + @DataProvider + public Object[][] provideData() { + return new Object[][]{ + {5.0, false}, + {10.0, true}, + {30.0, true}, + {40.0, false}, + {45.0, false}, + {50.0, false}, + {51.0, true}, + {99.0, true}, + {100.0, true}, + {101.0, false}, + {1001.0, false}, + }; + } + + @Test(dataProvider = "provideData") + public void testHook(Double cost, boolean success) + throws Exception { + when(ctx.getDriverQueryCost(driver)).thenReturn(new FactPartitionBasedQueryCost(cost)); + try { + hook.postEstimate(ctx); + assertTrue(success); + } catch (Exception e) { + assertFalse(success); + } + } +} http://git-wip-us.apache.org/repos/asf/lens/blob/02cbd6bc/lens-server-api/src/test/java/org/apache/lens/server/api/driver/hooks/UserBasedQueryHookTest.java ---------------------------------------------------------------------- diff --git a/lens-server-api/src/test/java/org/apache/lens/server/api/driver/hooks/UserBasedQueryHookTest.java b/lens-server-api/src/test/java/org/apache/lens/server/api/driver/hooks/UserBasedQueryHookTest.java new file mode 100644 index 0000000..47060f8 --- /dev/null +++ b/lens-server-api/src/test/java/org/apache/lens/server/api/driver/hooks/UserBasedQueryHookTest.java @@ -0,0 +1,87 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +/* + * + */ +package org.apache.lens.server.api.driver.hooks; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static org.testng.Assert.*; + +import org.apache.lens.server.api.driver.LensDriver; +import org.apache.lens.server.api.query.AbstractQueryContext; + +import org.apache.hadoop.conf.Configuration; + +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +public class UserBasedQueryHookTest { + + private LensDriver driver; + private Configuration conf; + private UserBasedQueryHook hook = new UserBasedQueryHook(); + private AbstractQueryContext ctx; + + @BeforeMethod + public void setUp() throws Exception { + driver = mock(LensDriver.class); + conf = new Configuration(); + when(driver.getConf()).thenReturn(conf); + ctx = mock(AbstractQueryContext.class); + } + + @DataProvider + public Object[][] provideData() { + return new Object[][]{ + {"a,b", "b,c", "d", false}, // disallow setting both + {"a,b", null, "c", false}, // disallowed + {null, "a,b", "c", true}, // not disallowed + {"a,b", null, "a", true}, // allowed + {null, "a,b", "a", false}, // not allowed + {null, null, "a", true}, // no restrictions + {"", "", "a", true}, // null and blank string are same + }; + } + + @Test(dataProvider = "provideData") + public void testPreRewrite(String allowedString, String disallowedString, String user, boolean success) + throws Exception { + if (allowedString != null) { + conf.set(UserBasedQueryHook.ALLOWED_USERS, allowedString); + } else { + conf.unset(UserBasedQueryHook.ALLOWED_USERS); + } + if (disallowedString != null) { + conf.set(UserBasedQueryHook.DISALLOWED_USERS, disallowedString); + } else { + conf.unset(UserBasedQueryHook.DISALLOWED_USERS); + } + when(ctx.getSubmittedUser()).thenReturn(user); + try { + hook.setDriver(driver); + hook.preRewrite(ctx); + assertTrue(success); + } catch (Exception e) { + assertFalse(success); + } + } +} http://git-wip-us.apache.org/repos/asf/lens/blob/02cbd6bc/lens-server-api/src/test/java/org/apache/lens/server/api/user/MockDriverQueryHook.java ---------------------------------------------------------------------- diff --git a/lens-server-api/src/test/java/org/apache/lens/server/api/user/MockDriverQueryHook.java b/lens-server-api/src/test/java/org/apache/lens/server/api/user/MockDriverQueryHook.java index f70979a..9dfb3b6 100644 --- a/lens-server-api/src/test/java/org/apache/lens/server/api/user/MockDriverQueryHook.java +++ b/lens-server-api/src/test/java/org/apache/lens/server/api/user/MockDriverQueryHook.java @@ -20,7 +20,7 @@ package org.apache.lens.server.api.user; import java.util.HashMap; -import org.apache.lens.server.api.driver.NoOpDriverQueryHook; +import org.apache.lens.server.api.driver.hooks.NoOpDriverQueryHook; import org.apache.lens.server.api.error.LensException; import org.apache.lens.server.api.query.AbstractQueryContext; import org.apache.lens.server.api.query.QueryContext; @@ -40,7 +40,7 @@ public class MockDriverQueryHook extends NoOpDriverQueryHook { public static final String POST_ESTIMATE = "POST_ESTIMATE"; @Override - public void preLaunch(QueryContext ctx) { + public void preLaunch(QueryContext ctx) throws LensException { super.preLaunch(ctx); ctx.getSelectedDriverConf().set(KEY_PRE_LAUNCH, VALUE_PRE_LAUNCH); } http://git-wip-us.apache.org/repos/asf/lens/blob/02cbd6bc/lens-server/src/main/resources/lensserver-default.xml ---------------------------------------------------------------------- diff --git a/lens-server/src/main/resources/lensserver-default.xml b/lens-server/src/main/resources/lensserver-default.xml index e652a0f..261fa52 100644 --- a/lens-server/src/main/resources/lensserver-default.xml +++ b/lens-server/src/main/resources/lensserver-default.xml @@ -938,6 +938,11 @@ </description> </property> <property> + <name>lens.server.query.cost.parser.class</name> + <value>org.apache.lens.server.api.query.cost.FactPartitionBasedQueryCost$Parser</value> + <description>The Query cost parser class. Default query cost class used is FactPartitionBasedQueryCost</description> + </property> + <property> <name>lens.cube.metastore.enable.datacompleteness.check</name> <value>false</value> <description>This property is to enable Data Completeness Checks while resolving partitions.</description> http://git-wip-us.apache.org/repos/asf/lens/blob/02cbd6bc/lens-server/src/test/resources/drivers/hive/hive1/hivedriver-site.xml ---------------------------------------------------------------------- diff --git a/lens-server/src/test/resources/drivers/hive/hive1/hivedriver-site.xml b/lens-server/src/test/resources/drivers/hive/hive1/hivedriver-site.xml index 50985b5..1f0ff43 100644 --- a/lens-server/src/test/resources/drivers/hive/hive1/hivedriver-site.xml +++ b/lens-server/src/test/resources/drivers/hive/hive1/hivedriver-site.xml @@ -76,7 +76,7 @@ </property> <property> - <name>lens.driver.hive.query.hook.class</name> + <name>lens.driver.hive.query.hook.classes</name> <value>org.apache.lens.server.api.user.MockDriverQueryHook</value> <description>The query hook class for hive driver. </description> http://git-wip-us.apache.org/repos/asf/lens/blob/02cbd6bc/lens-server/src/test/resources/drivers/hive/hive2/hivedriver-site.xml ---------------------------------------------------------------------- diff --git a/lens-server/src/test/resources/drivers/hive/hive2/hivedriver-site.xml b/lens-server/src/test/resources/drivers/hive/hive2/hivedriver-site.xml index b485100..eb902e0 100644 --- a/lens-server/src/test/resources/drivers/hive/hive2/hivedriver-site.xml +++ b/lens-server/src/test/resources/drivers/hive/hive2/hivedriver-site.xml @@ -76,7 +76,7 @@ </property> <property> - <name>lens.driver.hive.query.hook.class</name> + <name>lens.driver.hive.query.hook.classes</name> <value>org.apache.lens.server.api.user.MockDriverQueryHook</value> <description>The query hook class for hive driver. </description> http://git-wip-us.apache.org/repos/asf/lens/blob/02cbd6bc/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 29c59d3..8ea64b7 100644 --- a/pom.xml +++ b/pom.xml @@ -58,7 +58,7 @@ <commons.collections.version>3.2.1</commons.collections.version> <joda.time.version>2.0</joda.time.version> <quartz.version>2.2.2</quartz.version> - <guava.version>13.0.1</guava.version> + <guava.version>14.0.1</guava.version> <lombok.version>1.16.6</lombok.version> <lombok.maven.plugin.version>1.16.4.1</lombok.maven.plugin.version> <typesafe.config.version>1.2.1</typesafe.config.version>
