Repository: lens Updated Branches: refs/heads/master c7451f8e8 -> edcdd9685
LENS-910 : Add session config to skip filtering cube related tables from all the tables in a database Project: http://git-wip-us.apache.org/repos/asf/lens/repo Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/edcdd968 Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/edcdd968 Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/edcdd968 Branch: refs/heads/master Commit: edcdd968541b6de572b65abbe92b4f3601c2f7a3 Parents: c7451f8 Author: Deepak Barr <[email protected]> Authored: Tue Jan 12 12:24:29 2016 +0530 Committer: Deepak Kumar Barr <[email protected]> Committed: Tue Jan 12 12:24:29 2016 +0530 ---------------------------------------------------------------------- .../lens/cli/TestLensNativeTableCommands.java | 4 ++- .../lens/server/api/LensConfConstants.java | 10 ++++++++ .../metastore/CubeMetastoreServiceImpl.java | 6 +++++ .../src/main/resources/lenssession-default.xml | 6 +++++ .../apache/lens/server/LensServerTestUtil.java | 5 +++- .../server/metastore/TestMetastoreService.java | 26 +++++++++++++++++++- src/site/apt/admin/session-config.apt | 2 ++ 7 files changed, 56 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/lens/blob/edcdd968/lens-cli/src/test/java/org/apache/lens/cli/TestLensNativeTableCommands.java ---------------------------------------------------------------------- diff --git a/lens-cli/src/test/java/org/apache/lens/cli/TestLensNativeTableCommands.java b/lens-cli/src/test/java/org/apache/lens/cli/TestLensNativeTableCommands.java index d453803..e5f11f2 100644 --- a/lens-cli/src/test/java/org/apache/lens/cli/TestLensNativeTableCommands.java +++ b/lens-cli/src/test/java/org/apache/lens/cli/TestLensNativeTableCommands.java @@ -18,6 +18,8 @@ */ package org.apache.lens.cli; +import java.util.HashMap; + import org.apache.lens.cli.commands.LensNativeTableCommands; import org.apache.lens.client.LensClient; import org.apache.lens.server.LensServerTestUtil; @@ -50,7 +52,7 @@ public class TestLensNativeTableCommands extends LensCliApplicationTest { LOG.debug("Starting to test nativetable commands"); String tblList = command.showNativeTables(); Assert.assertFalse(tblList.contains("test_native_table_command")); - LensServerTestUtil.createHiveTable("test_native_table_command"); + LensServerTestUtil.createHiveTable("test_native_table_command", new HashMap<String, String>()); tblList = command.showNativeTables(); Assert.assertTrue(tblList.contains("test_native_table_command")); http://git-wip-us.apache.org/repos/asf/lens/blob/edcdd968/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java ---------------------------------------------------------------------- diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java b/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java index 88e5a01..a3dbfc0 100644 --- a/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java +++ b/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java @@ -996,4 +996,14 @@ public final class LensConfConstants { * The Constant DEFAULT_HDFS_OUTPUT_RETENTION. */ public static final String DEFAULT_HDFS_OUTPUT_RETENTION = "1 day"; + + /** + * The Constant EXCLUDE_CUBE_TABLES. + */ + public static final String EXCLUDE_CUBE_TABLES = SESSION_PFX + "metastore.exclude.cubetables.from.nativetables"; + + /** + * The Constant DEFAULT_EXCLUDE_CUBE_TABLES. + */ + public static final boolean DEFAULT_EXCLUDE_CUBE_TABLES = true; } http://git-wip-us.apache.org/repos/asf/lens/blob/edcdd968/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java ---------------------------------------------------------------------- diff --git a/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java b/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java index cf49a13..fc67df1 100644 --- a/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java +++ b/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java @@ -31,12 +31,14 @@ import org.apache.lens.cube.metadata.*; import org.apache.lens.cube.metadata.timeline.PartitionTimeline; import org.apache.lens.server.BaseLensService; import org.apache.lens.server.LensServerConf; +import org.apache.lens.server.api.LensConfConstants; import org.apache.lens.server.api.error.LensException; import org.apache.lens.server.api.health.HealthStatus; import org.apache.lens.server.api.metastore.CubeMetastoreService; import org.apache.lens.server.session.LensSessionImpl; import org.apache.commons.lang.StringUtils; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.metastore.IMetaStoreClient; import org.apache.hadoop.hive.metastore.api.*; import org.apache.hadoop.hive.ql.metadata.Hive; @@ -1193,6 +1195,10 @@ public class CubeMetastoreServiceImpl extends BaseLensService implements CubeMet msc = getSession(sessionid).getMetaStoreClient(); List<String> tables = msc.getAllTables( dbName); + Configuration conf = getSession(sessionid).getSessionConf(); + if (!conf.getBoolean(LensConfConstants.EXCLUDE_CUBE_TABLES, LensConfConstants.DEFAULT_EXCLUDE_CUBE_TABLES)) { + return tables; + } List<String> result = new ArrayList<String>(); if (tables != null && !tables.isEmpty()) { List<org.apache.hadoop.hive.metastore.api.Table> tblObjects = http://git-wip-us.apache.org/repos/asf/lens/blob/edcdd968/lens-server/src/main/resources/lenssession-default.xml ---------------------------------------------------------------------- diff --git a/lens-server/src/main/resources/lenssession-default.xml b/lens-server/src/main/resources/lenssession-default.xml index 52e0cd0..a321c3f 100644 --- a/lens-server/src/main/resources/lenssession-default.xml +++ b/lens-server/src/main/resources/lenssession-default.xml @@ -311,4 +311,10 @@ <description>Whether to fail the query of data is partial</description> </property> + <property> + <name>lens.session.metastore.exclude.cubetables.from.nativetables</name> + <value>true</value> + <description>Exclude cube related tables when fetching native tables</description> + </property> + </configuration> http://git-wip-us.apache.org/repos/asf/lens/blob/edcdd968/lens-server/src/test/java/org/apache/lens/server/LensServerTestUtil.java ---------------------------------------------------------------------- diff --git a/lens-server/src/test/java/org/apache/lens/server/LensServerTestUtil.java b/lens-server/src/test/java/org/apache/lens/server/LensServerTestUtil.java index 57bedee..94dd394 100644 --- a/lens-server/src/test/java/org/apache/lens/server/LensServerTestUtil.java +++ b/lens-server/src/test/java/org/apache/lens/server/LensServerTestUtil.java @@ -223,13 +223,16 @@ public final class LensServerTestUtil { * @param tableName the table name * @throws HiveException the hive exception */ - public static void createHiveTable(String tableName) throws HiveException { + public static void createHiveTable(String tableName, Map<String, String> parameters) throws HiveException { List<FieldSchema> columns = new ArrayList<FieldSchema>(); columns.add(new FieldSchema("col1", "string", "")); List<FieldSchema> partCols = new ArrayList<FieldSchema>(); partCols.add(new FieldSchema("pcol1", "string", "")); Map<String, String> params = new HashMap<String, String>(); params.put("test.hive.table.prop", "tvalue"); + if (null != parameters && !parameters.isEmpty()) { + params.putAll(parameters); + } Table tbl = Hive.get().newTable(tableName); tbl.setTableType(TableType.MANAGED_TABLE); tbl.getTTable().getSd().setCols(columns); http://git-wip-us.apache.org/repos/asf/lens/blob/edcdd968/lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java ---------------------------------------------------------------------- diff --git a/lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java b/lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java index b0044da..925fc86 100644 --- a/lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java +++ b/lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java @@ -50,6 +50,7 @@ import org.apache.lens.cube.metadata.ExprColumn.ExprSpec; import org.apache.lens.server.LensJerseyTest; import org.apache.lens.server.LensServerTestUtil; import org.apache.lens.server.LensServices; +import org.apache.lens.server.api.LensConfConstants; import org.apache.lens.server.api.metastore.CubeMetastoreService; import org.apache.lens.server.api.util.LensUtil; @@ -2415,7 +2416,7 @@ public class TestMetastoreService extends LensJerseyTest { // create hive table String tableName = "test_simple_table"; SessionState.get().setCurrentDatabase(DB); - LensServerTestUtil.createHiveTable(tableName); + LensServerTestUtil.createHiveTable(tableName, new HashMap<String, String>()); WebTarget target = target().path("metastore").path("nativetables"); // get all native tables @@ -2447,6 +2448,29 @@ public class TestMetastoreService extends LensJerseyTest { assertEquals(nativetables.getElements().size(), 1); assertEquals(nativetables.getElements().get(0), tableName); + // test for lens.session.metastore.exclude.cubetables.from.nativetables session config + String cubeTableName = "test_cube_table"; + Map<String, String> params = new HashMap<String, String>(); + params.put(MetastoreConstants.TABLE_TYPE_KEY, CubeTableType.CUBE.name()); + LensServerTestUtil.createHiveTable(cubeTableName, params); + + // Test for excluding cube tables + nativetables = target.queryParam("sessionid", lensSessionId).queryParam("dbName", DB) + .queryParam("dbOption", "current").request(mediaType).get(StringList.class); + assertEquals(nativetables.getElements().size(), 1); + assertEquals(nativetables.getElements().get(0), tableName); + + // Test for not excluding cube tables + Map<String, String> sessionConf = new HashMap<String, String>(); + sessionConf.put(LensConfConstants.EXCLUDE_CUBE_TABLES, "false"); + LensSessionHandle lensSessionId2 = + metastoreService.openSession("foo", "bar", sessionConf); + nativetables = target.queryParam("sessionid", lensSessionId2).queryParam("dbName", DB) + .queryParam("dbOption", "current").request(mediaType).get(StringList.class); + assertEquals(nativetables.getElements().size(), 2); + assertTrue(nativetables.getElements().contains(tableName)); + assertTrue(nativetables.getElements().contains(cubeTableName)); + // Now get the table JAXBElement<XNativeTable> actualElement = target.path(tableName).queryParam( "sessionid", lensSessionId).request(mediaType).get(new GenericType<JAXBElement<XNativeTable>>() {}); http://git-wip-us.apache.org/repos/asf/lens/blob/edcdd968/src/site/apt/admin/session-config.apt ---------------------------------------------------------------------- diff --git a/src/site/apt/admin/session-config.apt b/src/site/apt/admin/session-config.apt index 5ed51c3..05a2c2c 100644 --- a/src/site/apt/admin/session-config.apt +++ b/src/site/apt/admin/session-config.apt @@ -98,4 +98,6 @@ Lens session configuration *--+--+---+--+ |37|lens.session.loggedin.user| |The username used to log in to lens. e.g. LDAP user| *--+--+---+--+ +|38|lens.session.metastore.exclude.cubetables.from.nativetables|true|Exclude cube related tables when fetching native tables.| +*--+--+---+--+ The configuration parameters and their default values
