LENS-518 : Move Hive dependency to Apache Hive master
Project: http://git-wip-us.apache.org/repos/asf/lens/repo Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/d6b12169 Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/d6b12169 Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/d6b12169 Branch: refs/heads/master Commit: d6b1216922f1438a7fc000dfcdd9121d87c65149 Parents: d264c9a Author: Rajat Khandelwal <[email protected]> Authored: Mon May 9 16:45:32 2016 +0530 Committer: Amareshwari Sriramadasu <[email protected]> Committed: Mon May 9 16:45:32 2016 +0530 ---------------------------------------------------------------------- lens-cli/pom.xml | 31 +++ .../apache/lens/cli/TestLensQueryCommands.java | 28 ++- lens-client/pom.xml | 21 ++ lens-cube/pom.xml | 27 ++- .../lens/cube/metadata/CubeMetastoreClient.java | 11 +- .../org/apache/lens/cube/metadata/Storage.java | 4 +- .../lens/cube/parse/CubeQueryContext.java | 7 +- .../lens/cube/parse/CubeSemanticAnalyzer.java | 7 +- .../org/apache/lens/cube/parse/HQLParser.java | 35 ++-- .../cube/metadata/TestCubeMetastoreClient.java | 19 +- .../lens/cube/parse/TestBaseCubeQueries.java | 34 ++-- .../apache/lens/cube/parse/TestHQLParser.java | 56 +++--- .../org/apache/lens/cube/parse/TestQuery.java | 9 +- ...stFactPartitionBasedQueryCostCalculator.java | 2 +- lens-cube/src/test/resources/hive-site.xml | 10 + lens-driver-es/pom.xml | 129 +++++++----- .../client/jest/JestResultSetTransformer.java | 2 +- .../org/apache/lens/driver/es/ESDriverTest.java | 5 + .../org/apache/lens/driver/es/MockClientES.java | 2 +- .../driver/es/ResultSetTransformationTest.java | 2 +- lens-driver-es/src/test/resources/hive-site.xml | 53 +++++ lens-driver-es/src/test/resources/logback.xml | 36 ++++ lens-driver-hive/pom.xml | 17 ++ .../driver/hive/EmbeddedThriftConnection.java | 200 ++++++++++++++++++- .../org/apache/lens/driver/hive/HiveDriver.java | 62 +++--- .../lens/driver/hive/HiveInMemoryResultSet.java | 2 +- .../apache/lens/driver/hive/HiveQueryPlan.java | 63 ++++-- .../driver/hive/RemoteThriftConnection.java | 7 +- .../apache/lens/driver/hive/TestHiveDriver.java | 185 +++++++++-------- .../lens/driver/hive/TestRemoteHiveDriver.java | 79 ++++---- .../src/test/resources/hive-site.xml | 10 + lens-driver-jdbc/pom.xml | 24 +++ .../lens/driver/jdbc/ColumnarSQLRewriter.java | 2 +- .../apache/lens/driver/jdbc/JDBCResultSet.java | 6 +- .../driver/jdbc/TestColumnarSQLRewriter.java | 77 +------ .../src/test/resources/hive-site.xml | 15 ++ lens-examples/pom.xml | 1 - lens-ml-lib/pom.xml | 4 + .../java/org/apache/lens/ml/TestMLResource.java | 5 - .../java/org/apache/lens/ml/TestMLRunner.java | 5 - lens-query-lib/pom.xml | 8 + .../lens/lib/query/FilePersistentFormatter.java | 9 +- .../lib/query/MockLensResultSetMetadata.java | 41 ++-- lens-server-api/pom.xml | 12 ++ .../api/driver/LensResultSetMetadata.java | 4 +- .../save/param/ParameterDataTypeEncoder.java | 2 +- .../MappedDiagnosticLogSegregationContext.java | 7 + lens-server/pom.xml | 28 +++ .../org/apache/lens/server/BaseLensService.java | 25 +-- .../org/apache/lens/server/LensServices.java | 2 +- .../metastore/CubeMetastoreServiceImpl.java | 6 +- .../apache/lens/server/metastore/JAXBUtils.java | 4 - .../server/query/QueryExecutionServiceImpl.java | 1 + .../apache/lens/server/rewrite/RewriteUtil.java | 61 +++--- .../lens/server/session/HiveSessionService.java | 27 +-- .../lens/server/session/LensSessionImpl.java | 75 ++++--- .../log/StatisticsLogPartitionHandler.java | 2 +- .../apache/lens/server/TestServerRestart.java | 2 +- .../lens/server/common/RestAPITestUtil.java | 2 +- .../server/metastore/TestMetastoreService.java | 4 +- .../lens/server/query/TestQueryService.java | 8 +- .../lens/server/query/TestResultFormatting.java | 2 +- .../lens/server/rewrite/TestRewriting.java | 198 ++++++------------ .../server/session/TestSessionClassLoaders.java | 2 +- .../lens/server/session/TestSessionExpiry.java | 4 +- lens-server/src/test/resources/hive-site.xml | 15 ++ lens-storage-db/pom.xml | 8 + .../src/test/resources/hive-site.xml | 9 + pom.xml | 93 ++++++++- src/site/apt/admin/deployment.apt | 8 +- src/site/apt/developer/contribute.apt | 2 +- src/site/apt/lenshome/install-and-run.apt | 3 +- tools/conf/server/lens-site.xml | 5 + 73 files changed, 1268 insertions(+), 705 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/lens/blob/d6b12169/lens-cli/pom.xml ---------------------------------------------------------------------- diff --git a/lens-cli/pom.xml b/lens-cli/pom.xml index bfa6f06..6f0555c 100644 --- a/lens-cli/pom.xml +++ b/lens-cli/pom.xml @@ -65,6 +65,16 @@ <scope>test</scope> </dependency> <dependency> + <groupId>org.apache.hive</groupId> + <artifactId>hive-exec</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.apache.hive</groupId> + <artifactId>hive-metastore</artifactId> + <scope>test</scope> + </dependency> + <dependency> <groupId>org.testng</groupId> <artifactId>testng</artifactId> </dependency> @@ -122,6 +132,27 @@ <version>${project.version}</version> <scope>test</scope> </dependency> + <dependency> + <groupId>org.apache.hive</groupId> + <artifactId>hive-shims</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.datanucleus</groupId> + <artifactId>datanucleus-api-jdo</artifactId> + </dependency> + <dependency> + <groupId>org.datanucleus</groupId> + <artifactId>datanucleus-rdbms</artifactId> + </dependency> + <dependency> + <groupId>org.apache.derby</groupId> + <artifactId>derby</artifactId> + </dependency> + <dependency> + <groupId>org.datanucleus</groupId> + <artifactId>javax.jdo</artifactId> + </dependency> </dependencies> <build> http://git-wip-us.apache.org/repos/asf/lens/blob/d6b12169/lens-cli/src/test/java/org/apache/lens/cli/TestLensQueryCommands.java ---------------------------------------------------------------------- diff --git a/lens-cli/src/test/java/org/apache/lens/cli/TestLensQueryCommands.java b/lens-cli/src/test/java/org/apache/lens/cli/TestLensQueryCommands.java index 1dd2ffe..4d8341a 100644 --- a/lens-cli/src/test/java/org/apache/lens/cli/TestLensQueryCommands.java +++ b/lens-cli/src/test/java/org/apache/lens/cli/TestLensQueryCommands.java @@ -60,15 +60,6 @@ import lombok.extern.slf4j.Slf4j; @Slf4j public class TestLensQueryCommands extends LensCliApplicationTest { - /** The explain plan. */ - private static String explainPlan = "TOK_QUERY\n" + " TOK_FROM\n" + " TOK_TABREF\n" + " TOK_TABNAME\n" - + " local_dim_table\n" + " test_dim\n" + " TOK_INSERT\n" + " TOK_DESTINATION\n" - + " TOK_DIR\n" + " TOK_TMP_FILE\n" + " TOK_SELECT\n" + " TOK_SELEXPR\n" - + " .\n" + " TOK_TABLE_OR_COL\n" + " test_dim\n" - + " id\n" + " TOK_SELEXPR\n" + " .\n" + " TOK_TABLE_OR_COL\n" - + " test_dim\n" + " name\n" + " TOK_WHERE\n" + " =\n" - + " .\n" + " TOK_TABLE_OR_COL\n" + " test_dim\n" - + " dt\n" + " 'latest'"; private File resDir; @BeforeClass @@ -241,7 +232,7 @@ public class TestLensQueryCommands extends LensCliApplicationTest { assertEquals(result, "No prepared queries"); final String qh2 = qCom.explainAndPrepare(sql, "testPrepQuery3"); - assertTrue(qh2.contains(explainPlan)); + assertExplainOutput(qh2); String handles = qCom.getAllPreparedQueries("testPrepQuery3", "all", -1, Long.MAX_VALUE); assertFalse(handles.contains("No prepared queries"), handles); @@ -282,7 +273,7 @@ public class TestLensQueryCommands extends LensCliApplicationTest { String result = qCom.explainQuery(sql, null); log.debug(result); - assertTrue(result.contains(explainPlan)); + assertExplainOutput(result); closeClientConnection(qCom); } @@ -406,10 +397,8 @@ public class TestLensQueryCommands extends LensCliApplicationTest { } /** - * Sets the up. - * - * @param client the new up - * @throws Exception the exception + * Sets up query command instances and adds partitions to a table. + * @throws Exception */ @BeforeClass public void setup() throws Exception { @@ -577,4 +566,13 @@ public class TestLensQueryCommands extends LensCliApplicationTest { client.closeConnection(); } + + private void assertExplainOutput(String result) { + assertTrue(result.contains("Stage-0 is a root stage")); + assertTrue(result.contains("Partition Description")); + assertTrue(result.contains("dt latest")); + assertTrue(result.contains("Processor Tree")); + assertTrue(result.contains("TableScan")); + } + } http://git-wip-us.apache.org/repos/asf/lens/blob/d6b12169/lens-client/pom.xml ---------------------------------------------------------------------- diff --git a/lens-client/pom.xml b/lens-client/pom.xml index 2032cf5..d8f7d6c 100644 --- a/lens-client/pom.xml +++ b/lens-client/pom.xml @@ -63,6 +63,11 @@ </dependency> <dependency> <groupId>org.apache.hive</groupId> + <artifactId>hive-metastore</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.apache.hive</groupId> <artifactId>hive-service</artifactId> <scope>test</scope> </dependency> @@ -147,6 +152,22 @@ <artifactId>jcl-over-slf4j</artifactId> </dependency> <dependency> + <groupId>org.datanucleus</groupId> + <artifactId>datanucleus-api-jdo</artifactId> + </dependency> + <dependency> + <groupId>org.datanucleus</groupId> + <artifactId>datanucleus-rdbms</artifactId> + </dependency> + <dependency> + <groupId>org.apache.derby</groupId> + <artifactId>derby</artifactId> + </dependency> + <dependency> + <groupId>org.datanucleus</groupId> + <artifactId>javax.jdo</artifactId> + </dependency> + <dependency> <groupId>net.sf.opencsv</groupId> <artifactId>opencsv</artifactId> </dependency> http://git-wip-us.apache.org/repos/asf/lens/blob/d6b12169/lens-cube/pom.xml ---------------------------------------------------------------------- diff --git a/lens-cube/pom.xml b/lens-cube/pom.xml index 0533f44..ef432c2 100644 --- a/lens-cube/pom.xml +++ b/lens-cube/pom.xml @@ -46,9 +46,17 @@ </dependency> <dependency> <groupId>org.apache.hive</groupId> + <artifactId>hive-metastore</artifactId> + </dependency> + <dependency> + <groupId>org.apache.hive</groupId> <artifactId>hive-service</artifactId> </dependency> <dependency> + <groupId>org.apache.hive</groupId> + <artifactId>hive-shims</artifactId> + </dependency> + <dependency> <groupId>org.mockito</groupId> <artifactId>mockito-all</artifactId> </dependency> @@ -90,6 +98,23 @@ <version>${project.version}</version> <scope>test</scope> </dependency> + <dependency> + <groupId>org.datanucleus</groupId> + <artifactId>datanucleus-api-jdo</artifactId> + </dependency> + <dependency> + <groupId>org.datanucleus</groupId> + <artifactId>datanucleus-rdbms</artifactId> + </dependency> + <dependency> + <groupId>org.apache.derby</groupId> + <artifactId>derby</artifactId> + </dependency> + <dependency> + <groupId>org.datanucleus</groupId> + <artifactId>javax.jdo</artifactId> + </dependency> + </dependencies> <build> @@ -98,7 +123,7 @@ <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-surefire-plugin</artifactId> <configuration> - <argLine>-Xms256m -Xmx512m -XX:PermSize=256m -XX:MaxPermSize=256m</argLine> + <argLine>-Xms256m -Xmx512m -XX:PermSize=256m -XX:MaxPermSize=256m -Xss4m</argLine> </configuration> </plugin> </plugins> http://git-wip-us.apache.org/repos/asf/lens/blob/d6b12169/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java index 42a8eb2..c8190bc 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java @@ -39,7 +39,7 @@ import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.TableType; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.InvalidOperationException; -import org.apache.hadoop.hive.ql.io.HiveFileFormatUtils; +import org.apache.hadoop.hive.ql.io.HiveOutputFormat; import org.apache.hadoop.hive.ql.metadata.Hive; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.metadata.Partition; @@ -225,10 +225,7 @@ public class CubeMetastoreClient { partition.getTPartition().getSd().getSerdeInfo().getParameters()); latestPart.setLocation(partition.getLocation()); latestPart.setInputFormatClass(partition.getInputFormatClass()); - latestPart.setOutputFormatClass(partition.getOutputFormatClass()); - // the following is a fix because hive has a bug: https://issues.apache.org/jira/browse/HIVE-11278. - latestPart.getTPartition().getSd().setOutputFormat( - HiveFileFormatUtils.getOutputFormatSubstitute(partition.getOutputFormatClass(), false).getName()); + latestPart.setOutputFormatClass(partition.getOutputFormatClass().asSubclass(HiveOutputFormat.class)); latestPart.getTPartition().getSd().getSerdeInfo() .setSerializationLib(partition.getTPartition().getSd().getSerdeInfo().getSerializationLib()); latestParts.add(latestPart); @@ -2061,7 +2058,7 @@ public class CubeMetastoreClient { } hiveTable.getTTable().getParameters().putAll(cubeTable.getProperties()); try { - getClient().alterTable(table, hiveTable); + getClient().alterTable(table, hiveTable, null); } catch (InvalidOperationException e) { throw new HiveException(e); } @@ -2074,7 +2071,7 @@ public class CubeMetastoreClient { public void alterHiveTable(String table, Table hiveTable) throws HiveException { try { - getClient().alterTable(table, hiveTable); + getClient().alterTable(table, hiveTable, null); } catch (InvalidOperationException e) { throw new HiveException(e); } http://git-wip-us.apache.org/repos/asf/lens/blob/d6b12169/lens-cube/src/main/java/org/apache/lens/cube/metadata/Storage.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/Storage.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/Storage.java index 9318603..012081a 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/Storage.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/Storage.java @@ -370,7 +370,7 @@ public abstract class Storage extends AbstractCubeTable implements PartitionMeta */ public void updatePartition(Hive client, String fact, Partition partition) throws InvalidOperationException, HiveException { - client.alterPartition(MetastoreUtil.getFactOrDimtableStorageTableName(fact, getName()), partition); + client.alterPartition(MetastoreUtil.getFactOrDimtableStorageTableName(fact, getName()), partition, null); } /** @@ -385,7 +385,7 @@ public abstract class Storage extends AbstractCubeTable implements PartitionMeta throws InvalidOperationException, HiveException { boolean success = false; try { - client.alterPartitions(MetastoreUtil.getFactOrDimtableStorageTableName(fact, getName()), partitions); + client.alterPartitions(MetastoreUtil.getFactOrDimtableStorageTableName(fact, getName()), partitions, null); success = true; } finally { if (success) { http://git-wip-us.apache.org/repos/asf/lens/blob/d6b12169/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java index c9b1475..6f016f2 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java @@ -682,7 +682,7 @@ public class CubeQueryContext implements TrackQueriedColumns, QueryAST { } public void setLimitValue(Integer value) { - qb.getParseInfo().setDestLimit(getClause(), value); + qb.getParseInfo().setDestLimit(getClause(), 0, value); } private String getStorageStringWithAlias(CandidateFact fact, Map<Dimension, CandidateDim> dimsToQuery, String alias) { @@ -1144,12 +1144,11 @@ public class CubeQueryContext implements TrackQueriedColumns, QueryAST { } public String getInsertClause() { - String insertString = ""; ASTNode destTree = qb.getParseInfo().getDestForClause(clauseName); if (destTree != null && ((ASTNode) (destTree.getChild(0))).getToken().getType() != TOK_TMP_FILE) { - insertString = "INSERT OVERWRITE" + HQLParser.getString(qb.getParseInfo().getDestForClause(clauseName)); + return "INSERT OVERWRITE" + HQLParser.getString(destTree); } - return insertString; + return ""; } public void addExprToAlias(ASTNode expr, ASTNode alias) { http://git-wip-us.apache.org/repos/asf/lens/blob/d6b12169/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeSemanticAnalyzer.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeSemanticAnalyzer.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeSemanticAnalyzer.java index ebbe404..fc96055 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeSemanticAnalyzer.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeSemanticAnalyzer.java @@ -24,6 +24,7 @@ import java.util.List; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.ql.QueryState; import org.apache.hadoop.hive.ql.parse.*; import lombok.Getter; @@ -39,7 +40,7 @@ public class CubeSemanticAnalyzer extends SemanticAnalyzer { private QB cubeQB; public CubeSemanticAnalyzer(Configuration queryConf, HiveConf hiveConf) throws SemanticException { - super(hiveConf); + super(new QueryState(hiveConf)); this.queryConf = queryConf; this.hiveConf = hiveConf; setupRules(); @@ -52,7 +53,7 @@ public class CubeSemanticAnalyzer extends SemanticAnalyzer { @Override public void analyzeInternal(ASTNode ast) throws SemanticException { - reset(); + reset(true); cubeQB = new QB(null, null, false); if (ast.getToken().getType() == HiveParser.TOK_QUERY) { @@ -65,7 +66,7 @@ public class CubeSemanticAnalyzer extends SemanticAnalyzer { } } // analyzing from the ASTNode. - if (!doPhase1(ast, cubeQB, initPhase1Ctx())) { + if (!doPhase1(ast, cubeQB, initPhase1Ctx(), null)) { // if phase1Result false return return; } http://git-wip-us.apache.org/repos/asf/lens/blob/d6b12169/lens-cube/src/main/java/org/apache/lens/cube/parse/HQLParser.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/HQLParser.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/HQLParser.java index 1f6b66c..6c2a168 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/HQLParser.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/HQLParser.java @@ -545,17 +545,16 @@ public final class HQLParser { } } else if (TOK_DIR == rootType) { - buf.append(" directory "); + StringBuilder sb = new StringBuilder(); + boolean local = false; for (int i = 0; i < root.getChildCount(); i++) { - toInfixString((ASTNode) root.getChild(i), buf); - } - - } else if (TOK_LOCAL_DIR == rootType) { - buf.append(" local directory "); - for (int i = 0; i < root.getChildCount(); i++) { - toInfixString((ASTNode) root.getChild(i), buf); + if (root.getChild(i).getType() == KW_LOCAL) { + local = true; + } else { + toInfixString((ASTNode) root.getChild(i), sb); + } } - + buf.append(local ? " local": "").append(" directory ").append(sb); } else if (TOK_TAB == rootType) { buf.append(" table "); for (int i = 0; i < root.getChildCount(); i++) { @@ -766,8 +765,13 @@ public final class HQLParser { assert (node.getChildCount() != 0); if (node.getChild(0).getType() == HiveParser.Identifier) { String functionName = BaseSemanticAnalyzer.unescapeIdentifier(node.getChild(0).getText()); - if (FunctionRegistry.getGenericUDAFResolver(functionName) != null) { - return true; + try { + if (FunctionRegistry.getGenericUDAFResolver(functionName) != null) { + return true; + } + } catch (SemanticException e) { + log.error("Error trying to find whether {} is aggregate.", getString(node), e); + return false; } } } @@ -782,8 +786,13 @@ public final class HQLParser { assert (node.getChildCount() != 0); if (node.getChild(0).getType() == HiveParser.Identifier) { String functionName = BaseSemanticAnalyzer.unescapeIdentifier(node.getChild(0).getText()); - if (FunctionRegistry.getGenericUDAFResolver(functionName) == null) { - return true; + try { + if (FunctionRegistry.getGenericUDAFResolver(functionName) == null) { + return true; + } + } catch (SemanticException e) { + log.error("Error trying to find whether {} is udf node.", getString(node), e); + return false; } } } http://git-wip-us.apache.org/repos/asf/lens/blob/d6b12169/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java b/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java index 02571ee..1ed6258 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java @@ -1187,7 +1187,7 @@ public class TestCubeMetastoreClient { CubeFactTable factTable = new CubeFactTable(Hive.get(conf).getTable(factName)); factTable.alterColumn(new FieldSchema("testFactColAdd", "int", "test add column")); - factTable.alterColumn(new FieldSchema("msr3", "int", "test alter column")); + factTable.alterColumn(new FieldSchema("msr1", "float", "test alter column")); factTable.alterWeight(100L); Map<String, String> newProp = getHashMap("new.prop", "val"); factTable.addProperties(newProp); @@ -1210,13 +1210,18 @@ public class TestCubeMetastoreClient { assertTrue(altered.getUpdatePeriods().get(c2).contains(HOURLY)); assertTrue(altered.getCubeName().equalsIgnoreCase(CUBE_NAME.toLowerCase())); boolean contains = false; + boolean msr1Altered = false; for (FieldSchema column : altered.getColumns()) { if (column.getName().equals("testfactcoladd") && column.getType().equals("int")) { contains = true; break; } + if (column.getName().equals("msr1") && column.getType().equals("float")) { + msr1Altered = true; + } } - assertTrue(contains); + assertTrue(contains, "column did not get added"); + assertTrue(msr1Altered, "measure type did not get altered"); // alter storage table desc String c1TableName = getFactOrDimtableStorageTableName(factName, c1); @@ -2530,7 +2535,7 @@ public class TestCubeMetastoreClient { client.createCubeDimensionTable(zipDim.getName(), dimTblName, dimColumns, 100L, dumpPeriods, null, storageTables); CubeDimensionTable dimTable = client.getDimensionTable(dimTblName); - dimTable.alterColumn(new FieldSchema("testAddDim", "string", "test add column")); + dimTable.alterColumn(new FieldSchema("testAddDim", "int", "test add column")); List<CubeDimensionTable> tbls = client.getAllDimensionTables(zipDim); boolean found = false; @@ -2549,7 +2554,7 @@ public class TestCubeMetastoreClient { List<FieldSchema> columns = altered.getColumns(); boolean contains = false; for (FieldSchema column : columns) { - if (column.getName().equals("testadddim") && column.getType().equals("string")) { + if (column.getName().equals("testadddim") && column.getType().equals("int")) { contains = true; break; } @@ -2557,13 +2562,13 @@ public class TestCubeMetastoreClient { assertTrue(contains); // Test alter column - dimTable.alterColumn(new FieldSchema("testAddDim", "int", "change type")); + dimTable.alterColumn(new FieldSchema("testAddDim", "float", "change type")); client.alterCubeDimensionTable(dimTblName, dimTable, storageTables); altered = new CubeDimensionTable(Hive.get(conf).getTable(dimTblName)); boolean typeChanged = false; for (FieldSchema column : altered.getColumns()) { - if (column.getName().equals("testadddim") && column.getType().equals("int")) { + if (column.getName().equals("testadddim") && column.getType().equals("float")) { typeChanged = true; break; } @@ -2586,7 +2591,7 @@ public class TestCubeMetastoreClient { assertEquals(alteredC1Table.getInputFormatClass(), SequenceFileInputFormat.class); boolean storageTblColAltered = false; for (FieldSchema column : alteredC1Table.getAllCols()) { - if (column.getName().equals("testadddim") && column.getType().equals("int")) { + if (column.getName().equals("testadddim") && column.getType().equals("float")) { storageTblColAltered = true; break; } http://git-wip-us.apache.org/repos/asf/lens/blob/d6b12169/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBaseCubeQueries.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBaseCubeQueries.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBaseCubeQueries.java index bc4f2a1..d17c18f 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBaseCubeQueries.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBaseCubeQueries.java @@ -469,23 +469,23 @@ public class TestBaseCubeQueries extends TestQueryRewrite { @Test public void testMultiFactQueryWithExprOnDimsWithoutAliases() throws Exception { String hqlQuery = - rewrite("select func1(dim1), func2(dim1), msr12, roundedmsr2 from basecube where " + TWO_DAYS_RANGE, conf); + rewrite("select reverse(dim1), ltrim(dim1), msr12, roundedmsr2 from basecube where " + TWO_DAYS_RANGE, conf); String expected1 = - getExpectedQuery(cubeName, "select func1(basecube.dim1) as `expr1`, func2(basecube.dim1) as `expr2`," + getExpectedQuery(cubeName, "select reverse(basecube.dim1) as `expr1`, ltrim(basecube.dim1) as `expr2`," + " sum(basecube.msr12) as `msr12` FROM ", null, - " group by func1(basecube.dim1), func2(basecube.dim1)", + " group by reverse(basecube.dim1), ltrim(basecube.dim1)", getWhereForDailyAndHourly2days(cubeName, "C1_testFact2_BASE")); String expected2 = - getExpectedQuery(cubeName, "select func1(basecube.dim1) as `expr1`, func2(basecube.dim1) as `expr2`," + getExpectedQuery(cubeName, "select reverse(basecube.dim1) as `expr1`, ltrim(basecube.dim1) as `expr2`," + " round(sum(basecube.msr2)/1000) as `roundedmsr2` FROM ", null, - " group by func1(basecube.dim1), func2(basecube.dim1)", + " group by reverse(basecube.dim1), ltrim(basecube.dim1)", getWhereForDailyAndHourly2days(cubeName, "C1_testFact1_BASE")); compareContains(expected1, hqlQuery); compareContains(expected2, hqlQuery); - assertTrue(hqlQuery.toLowerCase().startsWith("select coalesce(mq1.expr1, mq2.expr1) `func1(dim1)`," - + " coalesce(mq1.expr2, mq2.expr2) `func2(dim1)`, mq2.msr12 msr12, mq1.roundedmsr2 roundedmsr2 from ") - || hqlQuery.toLowerCase().startsWith("select coalesce(mq1.expr1, mq2.expr1) `func1(dim1)`," - + " coalesce(mq1.expr2, mq2.expr2) `func2(dim1)`, mq1.msr12 msr12, mq2.roundedmsr2 roundedmsr2 from "), + assertTrue(hqlQuery.toLowerCase().startsWith("select coalesce(mq1.expr1, mq2.expr1) `reverse(dim1)`," + + " coalesce(mq1.expr2, mq2.expr2) `ltrim(dim1)`, mq2.msr12 msr12, mq1.roundedmsr2 roundedmsr2 from ") + || hqlQuery.toLowerCase().startsWith("select coalesce(mq1.expr1, mq2.expr1) `reverse(dim1)`," + + " coalesce(mq1.expr2, mq2.expr2) `ltrim(dim1)`, mq1.msr12 msr12, mq2.roundedmsr2 roundedmsr2 from "), hqlQuery); assertTrue(hqlQuery.contains("mq1 full outer join ") && hqlQuery.endsWith("mq2 on mq1.expr1 <=> mq2.expr1 AND mq1.expr2 <=> mq2.expr2"), hqlQuery); @@ -494,23 +494,23 @@ public class TestBaseCubeQueries extends TestQueryRewrite { @Test public void testMultiFactQueryWithDirectMsr() throws Exception { String hqlQuery = - rewrite("select func1(dim1), directMsrExpr as directMsr, roundedmsr2 from basecube where " + TWO_DAYS_RANGE, + rewrite("select reverse(dim1), directMsrExpr as directMsr, roundedmsr2 from basecube where " + TWO_DAYS_RANGE, conf); String expected1 = - getExpectedQuery(cubeName, "select func1(basecube.dim1) as `expr1`, max(basecube.msr13) + count(basecube . msr14)" - + " as `expr2` FROM ", null, - " group by func1(basecube.dim1)", getWhereForDailyAndHourly2days(cubeName, "C1_testFact3_BASE")); + getExpectedQuery(cubeName, "select reverse(basecube.dim1) as `expr1`, " + + "max(basecube.msr13) + count(basecube . msr14) as `expr2` FROM ", null, + " group by reverse(basecube.dim1)", getWhereForDailyAndHourly2days(cubeName, "C1_testFact3_BASE")); String expected2 = - getExpectedQuery(cubeName, "select func1(basecube.dim1) as expr1, round(sum(basecube.msr2)/1000) as `roundedmsr2`" - + " FROM ", null, " group by func1(basecube.dim1)", + getExpectedQuery(cubeName, "select reverse(basecube.dim1) as expr1, " + + "round(sum(basecube.msr2)/1000) as `roundedmsr2` FROM ", null, " group by reverse(basecube.dim1)", getWhereForDailyAndHourly2days(cubeName, "C1_testFact1_BASE")); compareContains(expected1, hqlQuery); compareContains(expected2, hqlQuery); assertTrue(hqlQuery.toLowerCase().startsWith( - "select coalesce(mq1.expr1, mq2.expr1) `func1(dim1)`, mq2.expr2 `directmsr`, mq1.roundedmsr2 roundedmsr2 " + "select coalesce(mq1.expr1, mq2.expr1) `reverse(dim1)`, mq2.expr2 `directmsr`, mq1.roundedmsr2 roundedmsr2 " + "from ") || hqlQuery.toLowerCase().startsWith( - "select coalesce(mq1.expr1, mq2.expr1) `func1(dim1)`, mq1.expr2 `directmsr`, mq2.roundedmsr2 roundedmsr2 " + "select coalesce(mq1.expr1, mq2.expr1) `reverse(dim1)`, mq1.expr2 `directmsr`, mq2.roundedmsr2 roundedmsr2 " + "from "), hqlQuery.toLowerCase()); assertTrue(hqlQuery.contains("mq1 full outer join ") && hqlQuery.endsWith("mq2 on mq1.expr1 <=> mq2.expr1"), http://git-wip-us.apache.org/repos/asf/lens/blob/d6b12169/lens-cube/src/test/java/org/apache/lens/cube/parse/TestHQLParser.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestHQLParser.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestHQLParser.java index e23290f..861d4f7 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestHQLParser.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestHQLParser.java @@ -31,6 +31,7 @@ import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.parse.ASTNode; import org.apache.hadoop.hive.ql.parse.HiveParser; import org.apache.hadoop.hive.ql.parse.ParseException; +import org.apache.hadoop.hive.ql.session.SessionState; import org.testng.Assert; import org.testng.annotations.DataProvider; @@ -42,6 +43,12 @@ import lombok.extern.slf4j.Slf4j; public class TestHQLParser { HiveConf conf = new HiveConf(); + + { + conf.setBoolVar(HiveConf.ConfVars.HIVE_SUPPORT_SQL11_RESERVED_KEYWORDS, false); + SessionState.start(conf); + } + @Test public void testGroupByOrderByGetString() throws Exception { String query = "SELECT a,b, sum(c) FROM tab GROUP BY a,f(b), d+e ORDER BY a, g(b), e/100"; @@ -216,35 +223,19 @@ public class TestHQLParser { @Test public void testOrderbyBrackets() throws Exception { - String query = "SELECT id from citytable order by ((citytable.id) asc)"; + String query = "SELECT id from citytable order by (citytable.id) asc"; // String hql = rewrite(driver, query); ASTNode tree = HQLParser.parseHQL(query, conf); ASTNode orderByTree = HQLParser.findNodeByPath(tree, TOK_INSERT, HiveParser.TOK_ORDERBY); String reconstructed = HQLParser.getString(orderByTree); System.out.println("RECONSTRUCTED0:" + reconstructed); - // Assert.assertEquals("(( citytable . id ) asc )", reconstructed); - HQLParser.parseHQL("SELECT citytable.id FROM citytable ORDER BY " + reconstructed, conf); - - String query2 = "SELECT id from citytable order by (citytable.id asc)"; - tree = HQLParser.parseHQL(query2, conf); - orderByTree = HQLParser.findNodeByPath(tree, TOK_INSERT, HiveParser.TOK_ORDERBY); - reconstructed = HQLParser.getString(orderByTree); - System.out.println("RECONSTRUCTED1:" + reconstructed); - HQLParser.parseHQL("SELECT citytable.id FROM citytable ORDER BY " + reconstructed, conf); - + Assert.assertEquals(reconstructed, "citytable.id asc"); String query3 = "SELECT id, name from citytable order by citytable.id asc, citytable.name desc"; tree = HQLParser.parseHQL(query3, conf); orderByTree = HQLParser.findNodeByPath(tree, TOK_INSERT, HiveParser.TOK_ORDERBY); reconstructed = HQLParser.getString(orderByTree); System.out.println("RECONSTRUCTED2:" + reconstructed); - HQLParser.parseHQL("SELECT id, name FROM citytable ORDER BY " + reconstructed, conf); - - String query4 = "SELECT id from citytable order by citytable.id"; - tree = HQLParser.parseHQL(query4, conf); - orderByTree = HQLParser.findNodeByPath(tree, TOK_INSERT, HiveParser.TOK_ORDERBY); - reconstructed = HQLParser.getString(orderByTree); - System.out.println("RECONSTRUCTED3:" + reconstructed); - HQLParser.parseHQL("SELECT citytable.id FROM citytable ORDER BY " + reconstructed, conf); + Assert.assertEquals(reconstructed, "citytable.id asc, citytable.name desc"); } @Test @@ -273,10 +264,10 @@ public class TestHQLParser { public void testAliasShouldBeQuoted() throws Exception { Assert.assertEquals(getSelectStrForQuery("select id as identity from sample_dim"), "id as `identity`"); Assert.assertEquals(getSelectStrForQuery("select id as `column identity` from sample_dim"), - "id as `column identity`"); + "id as `column identity`"); Assert.assertEquals(getSelectStrForQuery("select id identity from sample_dim"), "id as `identity`"); Assert.assertEquals(getSelectStrForQuery("select id `column identity` from sample_dim"), - "id as `column identity`"); + "id as `column identity`"); } private String getSelectStrForQuery(String query) throws Exception { @@ -387,7 +378,7 @@ public class TestHQLParser { @DataProvider public Object[][] nAryFlatteningDataProvider() { - return new Object[][] { + return new Object[][]{ {"a", "a"}, {"a or b", "a or b"}, {"a or b or c or d", "a or b or c or d"}, @@ -408,7 +399,7 @@ public class TestHQLParser { @DataProvider public Object[][] colsInExpr() { - return new Object[][] { + return new Object[][]{ {" t1.c1", new String[]{}}, // simple selection {" cie.c5", new String[]{"c5"}}, // simple selection {" fun1(cie.c4)", new String[]{"c4"}}, // simple selection @@ -436,7 +427,7 @@ public class TestHQLParser { @DataProvider public Object[][] primitiveBool() { - return new Object[][] { + return new Object[][]{ {" t1.c1", false}, {" t1.c1 = 24", true}, {" t1.c1 >= 24", true}, @@ -461,7 +452,7 @@ public class TestHQLParser { @DataProvider public Object[][] primitiveBoolFunc() { - return new Object[][] { + return new Object[][]{ {" t1.c1", false}, {" t1.c1 = 24", false}, {" t1.c1 >= 24", false}, @@ -483,4 +474,19 @@ public class TestHQLParser { boolean actual = HQLParser.isPrimitiveBooleanFunction(inputAST); Assert.assertEquals(actual, expected, "Received " + actual + " for input:" + input); } + + @DataProvider + public Object[][] dirDataProvider() { + return new Object[][]{ + {"directory 'a'"}, + {"local directory 'a'"}, + }; + } + + @Test(dataProvider = "dirDataProvider") + public void testLocalDirectory(String dirString) throws LensException { + String expr = "insert overwrite " + dirString + " select * from table"; + ASTNode tree = HQLParser.parseHQL(expr, conf); + Assert.assertEquals(HQLParser.getString((ASTNode) tree.getChild(1).getChild(0)), dirString); + } } http://git-wip-us.apache.org/repos/asf/lens/blob/d6b12169/lens-cube/src/test/java/org/apache/lens/cube/parse/TestQuery.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestQuery.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestQuery.java index b50ebd2..1dd353f 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestQuery.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestQuery.java @@ -225,13 +225,16 @@ public class TestQuery { } else if (expected.query == null) { return false; } - boolean equals = false; + return stringEquals(expected) || astEquals(expected); + } + + private boolean astEquals(TestQuery expected) { try { - equals = equalsAST(this.getAST(), expected.getAST()); + return equalsAST(this.getAST(), expected.getAST()); } catch (LensException e) { log.error("AST not valid", e); + return false; } - return equals || stringEquals(expected); } private boolean stringEquals(TestQuery expected) { http://git-wip-us.apache.org/repos/asf/lens/blob/d6b12169/lens-cube/src/test/java/org/apache/lens/cube/query/cost/TestFactPartitionBasedQueryCostCalculator.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/java/org/apache/lens/cube/query/cost/TestFactPartitionBasedQueryCostCalculator.java b/lens-cube/src/test/java/org/apache/lens/cube/query/cost/TestFactPartitionBasedQueryCostCalculator.java index 8a60224..262d452 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/query/cost/TestFactPartitionBasedQueryCostCalculator.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/query/cost/TestFactPartitionBasedQueryCostCalculator.java @@ -56,7 +56,7 @@ public class TestFactPartitionBasedQueryCostCalculator { when(driver.getConf()).thenReturn(new Configuration()); queryContext = mock(AbstractQueryContext.class); - ImmutableMap<String, Double> tableWeights = mock(ImmutableMap.class); + ImmutableMap<String, Double> tableWeights = new ImmutableMap.Builder<String, Double>().build(); FactPartition fp1 = mockFactPartition(DAILY, tableWeights, 0.7); FactPartition fp2 = mockFactPartition(HOURLY, tableWeights, 0.8); http://git-wip-us.apache.org/repos/asf/lens/blob/d6b12169/lens-cube/src/test/resources/hive-site.xml ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/resources/hive-site.xml b/lens-cube/src/test/resources/hive-site.xml index 8456f8b..d21e61c 100644 --- a/lens-cube/src/test/resources/hive-site.xml +++ b/lens-cube/src/test/resources/hive-site.xml @@ -36,6 +36,11 @@ </property> <property> + <name>hive.exec.scratchdir</name> + <value>${project.build.directory}/hive/scratch</value> + </property> + + <property> <name>hive.querylog.location</name> <value>${project.build.directory}/query_logs</value> <description>Location of the structured hive logs</description> @@ -47,4 +52,9 @@ <description>JDBC connect string for a JDBC metastore</description> </property> + <property> + <name>datanucleus.schema.autoCreateTables</name> + <value>true</value> + </property> + </configuration> http://git-wip-us.apache.org/repos/asf/lens/blob/d6b12169/lens-driver-es/pom.xml ---------------------------------------------------------------------- diff --git a/lens-driver-es/pom.xml b/lens-driver-es/pom.xml index 89736ab..b00e78b 100644 --- a/lens-driver-es/pom.xml +++ b/lens-driver-es/pom.xml @@ -19,60 +19,81 @@ under the License. --> -<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> - <modelVersion>4.0.0</modelVersion> - <parent> - <groupId>org.apache.lens</groupId> - <artifactId>apache-lens</artifactId> - <version>2.6.0-beta-SNAPSHOT</version> - </parent> +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>org.apache.lens</groupId> + <artifactId>apache-lens</artifactId> + <version>2.6.0-beta-SNAPSHOT</version> + </parent> - <artifactId>lens-driver-es</artifactId> - <packaging>jar</packaging> - <description>ES execution driver</description> + <artifactId>lens-driver-es</artifactId> + <packaging>jar</packaging> + <description>ES execution driver</description> - <name>Lens Elastic Search Driver</name> - <properties> - <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> - </properties> - <dependencies> - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-client</artifactId> - </dependency> - <dependency> - <groupId>org.apache.hive</groupId> - <artifactId>hive-service</artifactId> - </dependency> - <dependency> - <groupId>org.apache.lens</groupId> - <artifactId>lens-cube</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <artifactId>lens-api</artifactId> - <groupId>org.apache.lens</groupId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>com.fasterxml.jackson.datatype</groupId> - <artifactId>jackson-datatype-guava</artifactId> - </dependency> - <dependency> - <groupId>org.testng</groupId> - <artifactId>testng</artifactId> - </dependency> - <dependency> - <groupId>io.searchbox</groupId> - <artifactId>jest</artifactId> - </dependency> - <dependency> - <groupId>org.apache.httpcomponents</groupId> - <artifactId>httpclient</artifactId> - </dependency> - <dependency> - <groupId>org.apache.httpcomponents</groupId> - <artifactId>httpcore</artifactId> - </dependency> - </dependencies> + <name>Lens Elastic Search Driver</name> + <properties> + <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> + </properties> + <dependencies> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-client</artifactId> + </dependency> + <dependency> + <groupId>org.apache.hive</groupId> + <artifactId>hive-service</artifactId> + </dependency> + <dependency> + <groupId>org.apache.hive</groupId> + <artifactId>hive-exec</artifactId> + </dependency> + <dependency> + <groupId>org.apache.lens</groupId> + <artifactId>lens-cube</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <artifactId>lens-api</artifactId> + <groupId>org.apache.lens</groupId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.fasterxml.jackson.datatype</groupId> + <artifactId>jackson-datatype-guava</artifactId> + </dependency> + <dependency> + <groupId>org.testng</groupId> + <artifactId>testng</artifactId> + </dependency> + <dependency> + <groupId>io.searchbox</groupId> + <artifactId>jest</artifactId> + </dependency> + <dependency> + <groupId>org.apache.httpcomponents</groupId> + <artifactId>httpclient</artifactId> + </dependency> + <dependency> + <groupId>org.apache.httpcomponents</groupId> + <artifactId>httpcore</artifactId> + </dependency> + <dependency> + <groupId>org.datanucleus</groupId> + <artifactId>datanucleus-api-jdo</artifactId> + </dependency> + <dependency> + <groupId>org.datanucleus</groupId> + <artifactId>datanucleus-rdbms</artifactId> + </dependency> + <dependency> + <groupId>org.apache.derby</groupId> + <artifactId>derby</artifactId> + </dependency> + <dependency> + <groupId>org.datanucleus</groupId> + <artifactId>javax.jdo</artifactId> + </dependency> + </dependencies> </project> http://git-wip-us.apache.org/repos/asf/lens/blob/d6b12169/lens-driver-es/src/main/java/org/apache/lens/driver/es/client/jest/JestResultSetTransformer.java ---------------------------------------------------------------------- diff --git a/lens-driver-es/src/main/java/org/apache/lens/driver/es/client/jest/JestResultSetTransformer.java b/lens-driver-es/src/main/java/org/apache/lens/driver/es/client/jest/JestResultSetTransformer.java index 38d91f9..d8644ba 100644 --- a/lens-driver-es/src/main/java/org/apache/lens/driver/es/client/jest/JestResultSetTransformer.java +++ b/lens-driver-es/src/main/java/org/apache/lens/driver/es/client/jest/JestResultSetTransformer.java @@ -26,8 +26,8 @@ import org.apache.lens.driver.es.client.ESResultSet; import org.apache.lens.server.api.driver.LensResultSetMetadata; import org.apache.commons.lang3.Validate; +import org.apache.hadoop.hive.serde2.thrift.Type; import org.apache.hive.service.cli.ColumnDescriptor; -import org.apache.hive.service.cli.Type; import org.apache.hive.service.cli.TypeDescriptor; import com.google.common.collect.Lists; http://git-wip-us.apache.org/repos/asf/lens/blob/d6b12169/lens-driver-es/src/test/java/org/apache/lens/driver/es/ESDriverTest.java ---------------------------------------------------------------------- diff --git a/lens-driver-es/src/test/java/org/apache/lens/driver/es/ESDriverTest.java b/lens-driver-es/src/test/java/org/apache/lens/driver/es/ESDriverTest.java index ab6f22b..61ee84a 100644 --- a/lens-driver-es/src/test/java/org/apache/lens/driver/es/ESDriverTest.java +++ b/lens-driver-es/src/test/java/org/apache/lens/driver/es/ESDriverTest.java @@ -21,6 +21,8 @@ package org.apache.lens.driver.es; import org.apache.lens.server.api.error.LensException; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.ql.session.SessionState; import org.testng.annotations.BeforeTest; @@ -37,6 +39,9 @@ public abstract class ESDriverTest { esDriverConfig = new ESDriverConfig(config); driver.configure(config, "es", "es1"); mockClientES = (MockClientES) driver.getESClient(); + HiveConf conf = new HiveConf(); + conf.addResource(config); + SessionState.start(conf); } protected abstract void initializeConfig(Configuration config); http://git-wip-us.apache.org/repos/asf/lens/blob/d6b12169/lens-driver-es/src/test/java/org/apache/lens/driver/es/MockClientES.java ---------------------------------------------------------------------- diff --git a/lens-driver-es/src/test/java/org/apache/lens/driver/es/MockClientES.java b/lens-driver-es/src/test/java/org/apache/lens/driver/es/MockClientES.java index 77300f9..27b577a 100644 --- a/lens-driver-es/src/test/java/org/apache/lens/driver/es/MockClientES.java +++ b/lens-driver-es/src/test/java/org/apache/lens/driver/es/MockClientES.java @@ -26,8 +26,8 @@ import org.apache.lens.driver.es.client.ESResultSet; import org.apache.lens.server.api.driver.LensResultSetMetadata; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.serde2.thrift.Type; import org.apache.hive.service.cli.ColumnDescriptor; -import org.apache.hive.service.cli.Type; import org.apache.hive.service.cli.TypeDescriptor; import com.google.common.collect.ImmutableMap; http://git-wip-us.apache.org/repos/asf/lens/blob/d6b12169/lens-driver-es/src/test/java/org/apache/lens/driver/es/ResultSetTransformationTest.java ---------------------------------------------------------------------- diff --git a/lens-driver-es/src/test/java/org/apache/lens/driver/es/ResultSetTransformationTest.java b/lens-driver-es/src/test/java/org/apache/lens/driver/es/ResultSetTransformationTest.java index 0b78639..bb8638d 100644 --- a/lens-driver-es/src/test/java/org/apache/lens/driver/es/ResultSetTransformationTest.java +++ b/lens-driver-es/src/test/java/org/apache/lens/driver/es/ResultSetTransformationTest.java @@ -31,8 +31,8 @@ import org.apache.lens.server.api.driver.LensResultSetMetadata; import org.apache.lens.server.api.error.LensException; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.serde2.thrift.Type; import org.apache.hive.service.cli.ColumnDescriptor; -import org.apache.hive.service.cli.Type; import org.apache.hive.service.cli.TypeDescriptor; import org.testng.annotations.BeforeTest; http://git-wip-us.apache.org/repos/asf/lens/blob/d6b12169/lens-driver-es/src/test/resources/hive-site.xml ---------------------------------------------------------------------- diff --git a/lens-driver-es/src/test/resources/hive-site.xml b/lens-driver-es/src/test/resources/hive-site.xml new file mode 100644 index 0000000..e49c876 --- /dev/null +++ b/lens-driver-es/src/test/resources/hive-site.xml @@ -0,0 +1,53 @@ +<?xml version="1.0"?> +<!-- + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +--> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> + +<configuration> + + <property> + <name>hive.metastore.warehouse.dir</name> + <value>${project.build.directory}/hive/warehouse</value> + </property> + + <property> + <name>hive.exec.scratchdir</name> + <value>${project.build.directory}/hive/scratch</value> + </property> + + <property> + <name>hive.querylog.location</name> + <value>${project.build.directory}/query_logs</value> + <description>Location of the structured hive logs</description> + </property> + + <property> + <name>javax.jdo.option.ConnectionURL</name> + <value>jdbc:derby:;databaseName=target/metastore_db;create=true</value> + <description>JDBC connect string for a JDBC metastore</description> + </property> + + <property> + <name>datanucleus.schema.autoCreateTables</name> + <value>true</value> + </property> + +</configuration> http://git-wip-us.apache.org/repos/asf/lens/blob/d6b12169/lens-driver-es/src/test/resources/logback.xml ---------------------------------------------------------------------- diff --git a/lens-driver-es/src/test/resources/logback.xml b/lens-driver-es/src/test/resources/logback.xml new file mode 100644 index 0000000..c32886d --- /dev/null +++ b/lens-driver-es/src/test/resources/logback.xml @@ -0,0 +1,36 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +--> +<configuration> + <appender name="TEST_LOG_FILE" class="ch.qos.logback.core.FileAppender"> + <file>target/test.log</file> + <encoder> + <pattern>%d{dd MMM yyyy HH:mm:ss,SSS} [%t] %-5p %c %L - %m%n</pattern> + </encoder> + <filter class="ch.qos.logback.classic.filter.ThresholdFilter"> + <level>INFO</level> + </filter> + </appender> + <root level="INFO"> + <appender-ref ref="TEST_LOG_FILE"/> + </root> +</configuration> + http://git-wip-us.apache.org/repos/asf/lens/blob/d6b12169/lens-driver-hive/pom.xml ---------------------------------------------------------------------- diff --git a/lens-driver-hive/pom.xml b/lens-driver-hive/pom.xml index e80fab8..9809953 100644 --- a/lens-driver-hive/pom.xml +++ b/lens-driver-hive/pom.xml @@ -43,6 +43,7 @@ <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-surefire-plugin</artifactId> <configuration> + <argLine>-Xms256m -Xmx512m -XX:PermSize=256m -XX:MaxPermSize=256m</argLine> <systemPropertyVariables> <hadoop.bin.path>${pom.basedir}/testutils/hadoop</hadoop.bin.path> </systemPropertyVariables> @@ -125,6 +126,22 @@ <version>${project.version}</version> <scope>test</scope> </dependency> + <dependency> + <groupId>org.datanucleus</groupId> + <artifactId>datanucleus-api-jdo</artifactId> + </dependency> + <dependency> + <groupId>org.datanucleus</groupId> + <artifactId>datanucleus-rdbms</artifactId> + </dependency> + <dependency> + <groupId>org.apache.derby</groupId> + <artifactId>derby</artifactId> + </dependency> + <dependency> + <groupId>org.datanucleus</groupId> + <artifactId>javax.jdo</artifactId> + </dependency> </dependencies> </project> http://git-wip-us.apache.org/repos/asf/lens/blob/d6b12169/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/EmbeddedThriftConnection.java ---------------------------------------------------------------------- diff --git a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/EmbeddedThriftConnection.java b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/EmbeddedThriftConnection.java index 0b46485..5f7252a 100644 --- a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/EmbeddedThriftConnection.java +++ b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/EmbeddedThriftConnection.java @@ -18,24 +18,200 @@ */ package org.apache.lens.driver.hive; -import java.io.IOException; +import java.util.List; +import java.util.Map; import org.apache.lens.server.api.error.LensException; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.ql.session.SessionState; +import org.apache.hive.service.auth.HiveAuthFactory; +import org.apache.hive.service.cli.*; import org.apache.hive.service.cli.thrift.EmbeddedThriftBinaryCLIService; import org.apache.hive.service.cli.thrift.ThriftCLIServiceClient; +import org.apache.hive.service.rpc.thrift.TCLIService; /** * The Class EmbeddedThriftConnection. */ public class EmbeddedThriftConnection implements ThriftConnection { + public static class SessionStateContext implements AutoCloseable { + /** + * This is needed because we're using embedded mode. In opening a hive session, a new session state is started + * and previous session state is lost, since it's all happening in the same jvm. + * For all other session operations (getting status, getting result etc), + * they are wrapped in acquire-release block in HiveSessionImpl, + * and the release clears session state for the current thread. + * Since it's happening in a single thread, the session is cleared for further operations too + * and needs to be restored for tests to proceed further. + */ + private SessionState state = SessionState.get(); + + @Override + public void close() { + if (state != null && !state.equals(SessionState.get())) { + SessionState.setCurrentSessionState(state); + } + } + } + //SUSPEND CHECKSTYLE CHECK InnerAssignmentCheck + public static class EmbeddedThriftCLIServiceClient extends ThriftCLIServiceClient { + + public EmbeddedThriftCLIServiceClient(TCLIService.Iface cliService) { + super(cliService); + } + + public SessionHandle openSession(String username, String password, + Map<String, String> configuration) throws HiveSQLException { + try (SessionStateContext ignored = new SessionStateContext()) { + return super.openSession(username, password, configuration); + } + } + + public SessionHandle openSessionWithImpersonation(String username, String password, + Map<String, String> configuration, String delegationToken) throws HiveSQLException { + try (SessionStateContext ignored = new SessionStateContext()) { + return super.openSessionWithImpersonation(username, password, configuration, delegationToken); + } + } + + public void closeSession(SessionHandle sessionHandle) throws HiveSQLException { + try (SessionStateContext ignored = new SessionStateContext()) { + super.closeSession(sessionHandle); + } + } + + public GetInfoValue getInfo(SessionHandle sessionHandle, GetInfoType infoType) throws HiveSQLException { + try (SessionStateContext ignored = new SessionStateContext()) { + return super.getInfo(sessionHandle, infoType); + } + } + + public OperationHandle executeStatement(SessionHandle sessionHandle, String statement, + Map<String, String> confOverlay) throws HiveSQLException { + try (SessionStateContext ignored = new SessionStateContext()) { + return super.executeStatement(sessionHandle, statement, confOverlay); + } + } + + public OperationHandle executeStatementAsync(SessionHandle sessionHandle, + String statement, Map<String, String> confOverlay) throws HiveSQLException { + try (SessionStateContext ignored = new SessionStateContext()) { + return super.executeStatementAsync(sessionHandle, statement, confOverlay); + } + } + + public OperationHandle getTypeInfo(SessionHandle sessionHandle) throws HiveSQLException { + try (SessionStateContext ignored = new SessionStateContext()) { + return super.getTypeInfo(sessionHandle); + } + } + + public OperationHandle getCatalogs(SessionHandle sessionHandle) throws HiveSQLException { + try (SessionStateContext ignored = new SessionStateContext()) { + return super.getCatalogs(sessionHandle); + } + } + + public OperationHandle getSchemas(SessionHandle sessionHandle, + String catalogName, String schemaName) throws HiveSQLException { + try (SessionStateContext ignored = new SessionStateContext()) { + return super.getSchemas(sessionHandle, catalogName, schemaName); + } + } + + public OperationHandle getTables(SessionHandle sessionHandle, + String catalogName, String schemaName, String tableName, List<String> tableTypes) throws HiveSQLException { + try (SessionStateContext ignored = new SessionStateContext()) { + return super.getTables(sessionHandle, catalogName, schemaName, tableName, tableTypes); + } + } + + public OperationHandle getTableTypes(SessionHandle sessionHandle) throws HiveSQLException { + try (SessionStateContext ignored = new SessionStateContext()) { + return super.getTableTypes(sessionHandle); + } + } + + public OperationHandle getColumns(SessionHandle sessionHandle, + String catalogName, String schemaName, String tableName, String columnName) throws HiveSQLException { + try (SessionStateContext ignored = new SessionStateContext()) { + return super.getColumns(sessionHandle, catalogName, schemaName, tableName, columnName); + } + } + + public OperationHandle getFunctions(SessionHandle sessionHandle, + String catalogName, String schemaName, String functionName) throws HiveSQLException { + try (SessionStateContext ignored = new SessionStateContext()) { + return super.getFunctions(sessionHandle, catalogName, schemaName, functionName); + } + } + + public void cancelOperation(OperationHandle opHandle) throws HiveSQLException { + try (SessionStateContext ignored = new SessionStateContext()) { + super.cancelOperation(opHandle); + } + } + + public void closeOperation(OperationHandle opHandle) throws HiveSQLException { + try (SessionStateContext ignored = new SessionStateContext()) { + super.closeOperation(opHandle); + } + } + + public TableSchema getResultSetMetadata(OperationHandle opHandle) throws HiveSQLException { + try (SessionStateContext ignored = new SessionStateContext()) { + return super.getResultSetMetadata(opHandle); + } + } + + public RowSet fetchResults(OperationHandle opHandle) throws HiveSQLException { + try (SessionStateContext ignored = new SessionStateContext()) { + return super.fetchResults(opHandle); + } + } + + public RowSet fetchResults(OperationHandle opHandle, FetchOrientation orientation, + long maxRows, FetchType fetchType) throws HiveSQLException { + try (SessionStateContext ignored = new SessionStateContext()) { + return super.fetchResults(opHandle, orientation, maxRows, fetchType); + } + } + + public String getDelegationToken(SessionHandle sessionHandle, HiveAuthFactory authFactory, + String owner, String renewer) throws HiveSQLException { + try (SessionStateContext ignored = new SessionStateContext()) { + return super.getDelegationToken(sessionHandle, authFactory, owner, renewer); + } + } + + public void cancelDelegationToken(SessionHandle sessionHandle, HiveAuthFactory authFactory, + String tokenStr) throws HiveSQLException { + try (SessionStateContext ignored = new SessionStateContext()) { + super.cancelDelegationToken(sessionHandle, authFactory, tokenStr); + } + } + + public void renewDelegationToken(SessionHandle sessionHandle, HiveAuthFactory authFactory, + String tokenStr) throws HiveSQLException { + try (SessionStateContext ignored = new SessionStateContext()) { + super.renewDelegationToken(sessionHandle, authFactory, tokenStr); + } + } + + @Override + public SessionHandle openSession(String username, String password) throws HiveSQLException { + try (SessionStateContext ignored = new SessionStateContext()) { + return super.openSession(username, password); + } + } + } /** The client. */ - private ThriftCLIServiceClient client; + private EmbeddedThriftCLIServiceClient client; /** The connected. */ - private boolean connected; + private EmbeddedThriftBinaryCLIService service; /* * (non-Javadoc) @@ -44,24 +220,34 @@ public class EmbeddedThriftConnection implements ThriftConnection { */ @Override public ThriftCLIServiceClient getClient() throws LensException { - if (!connected) { - client = new ThriftCLIServiceClient(new EmbeddedThriftBinaryCLIService()); - connected = true; + if (client == null) { + client = new EmbeddedThriftCLIServiceClient(getService()); } return client; } + private EmbeddedThriftBinaryCLIService getService() { + if (service == null) { + service = new EmbeddedThriftBinaryCLIService(); + } + return service; + } + /* * (non-Javadoc) * * @see java.io.Closeable#close() */ @Override - public void close() throws IOException { + public void close() { // Does nothing } @Override public void init(HiveConf conf, String user) { + try (SessionStateContext ignored = new SessionStateContext()) { + getService().init(conf); + } } + //RESUME CHECKSTYLE CHECK InnerAssignmentCheck } http://git-wip-us.apache.org/repos/asf/lens/blob/d6b12169/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java ---------------------------------------------------------------------- diff --git a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java index 19a010e..1463cc2 100644 --- a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java +++ b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java @@ -25,10 +25,7 @@ import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.ObjectInput; import java.io.ObjectOutput; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.concurrent.*; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.locks.Lock; @@ -61,17 +58,18 @@ import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.ql.TaskStatus; +import org.apache.hadoop.hive.ql.QueryDisplay.TaskDisplay; +import org.apache.hadoop.hive.ql.exec.Task; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hive.service.cli.*; -import org.apache.hive.service.cli.thrift.TOperationHandle; -import org.apache.hive.service.cli.thrift.TProtocolVersion; -import org.apache.hive.service.cli.thrift.TSessionHandle; - -import org.codehaus.jackson.map.ObjectMapper; -import org.codehaus.jackson.type.TypeReference; +import org.apache.hive.service.rpc.thrift.TOperationHandle; +import org.apache.hive.service.rpc.thrift.TProtocolVersion; +import org.apache.hive.service.rpc.thrift.TSessionHandle; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableSet; import lombok.Getter; import lombok.extern.slf4j.Slf4j; @@ -326,10 +324,10 @@ public class HiveDriver extends AbstractLensDriver { */ public HiveDriver() throws LensException { this.sessionLock = new ReentrantLock(); - lensToHiveSession = new HashMap<String, SessionHandle>(); - opHandleToSession = new ConcurrentHashMap<OperationHandle, SessionHandle>(); - orphanedHiveSessions = new ConcurrentLinkedQueue<SessionHandle>(); - resourcesAddedForSession = new HashMap<SessionHandle, Boolean>(); + lensToHiveSession = new HashMap<>(); + opHandleToSession = new ConcurrentHashMap<>(); + orphanedHiveSessions = new ConcurrentLinkedQueue<>(); + resourcesAddedForSession = new HashMap<>(); connectionExpiryThread.setDaemon(true); connectionExpiryThread.setName("HiveDriver-ConnectionExpiryThread"); connectionExpiryThread.start(); @@ -515,15 +513,13 @@ public class HiveDriver extends AbstractLensDriver { Configuration qdconf = ctx.getDriverConf(this); qdconf.set("mapred.job.name", ctx.getQueryHandle().toString()); SessionHandle sessionHandle = getSession(ctx); - op = getClient().executeStatement(sessionHandle, ctx.getSelectedDriverQuery(), - qdconf.getValByRegex(".*")); + op = getClient().executeStatement(sessionHandle, ctx.getSelectedDriverQuery(), qdconf.getValByRegex(".*")); log.info("The hive operation handle: {}", op); ctx.setDriverOpHandle(op.toString()); hiveHandles.put(ctx.getQueryHandle(), op); opHandleToSession.put(op, sessionHandle); updateStatus(ctx); OperationStatus status = getClient().getOperationStatus(op); - if (status.getState() == OperationState.ERROR) { throw new LensException("Unknown error while running query " + ctx.getUserQuery()); } @@ -597,9 +593,9 @@ public class HiveDriver extends AbstractLensDriver { try { // Get operation status from hive server log.debug("GetStatus hiveHandle: {}", hiveHandle); + fetchLogs(hiveHandle); OperationStatus opStatus = getClient().getOperationStatus(hiveHandle); log.debug("GetStatus on hiveHandle: {} returned state:", hiveHandle, opStatus.getState().name()); - switch (opStatus.getState()) { case CANCELED: context.getDriverStatus().setState(DriverQueryState.CANCELED); @@ -643,17 +639,18 @@ public class HiveDriver extends AbstractLensDriver { String errorMsg = null; if (StringUtils.isNotBlank(jsonTaskStatus)) { ObjectMapper mapper = new ObjectMapper(); + mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); in = new ByteArrayInputStream(jsonTaskStatus.getBytes("UTF-8")); - List<TaskStatus> taskStatuses = mapper.readValue(in, new TypeReference<List<TaskStatus>>() { + List<TaskDisplay> taskStatuses = mapper.readValue(in, new TypeReference<List<TaskDisplay>>() { }); int completedTasks = 0; StringBuilder errorMessage = new StringBuilder(); - for (TaskStatus taskStat : taskStatuses) { - String tstate = taskStat.getTaskState(); - if ("FINISHED_STATE".equalsIgnoreCase(tstate)) { + for (TaskDisplay taskStat : taskStatuses) { + Task.TaskState tstate = taskStat.taskState; + if (tstate == Task.TaskState.FINISHED) { completedTasks++; } - if ("FAILED_STATE".equalsIgnoreCase(tstate)) { + if (taskStat.getErrorMsg() != null) { appendTaskIds(errorMessage, taskStat); errorMessage.append(" has failed! "); } @@ -689,15 +686,27 @@ public class HiveDriver extends AbstractLensDriver { } } + private void fetchLogs(OperationHandle opHandle) throws LensException { + try { + for (Object[] o : getClient().fetchResults(opHandle, FetchOrientation.FETCH_NEXT, -1, FetchType.LOG)) { + for (Object logLine : o) { + log.info("Update from hive: " + String.valueOf(logLine)); + } + } + } catch (HiveSQLException e) { + log.error("Error fetching hive operation logs for {}", opHandle, e); + } + } + /** * Append task ids. * * @param message the message * @param taskStat the task stat */ - private void appendTaskIds(StringBuilder message, TaskStatus taskStat) { + private void appendTaskIds(StringBuilder message, TaskDisplay taskStat) { message.append(taskStat.getTaskId()).append("("); - message.append(taskStat.getType()).append("):"); + message.append(taskStat.getTaskType()).append("):"); if (taskStat.getExternalHandle() != null) { message.append(taskStat.getExternalHandle()).append(":"); } @@ -728,6 +737,7 @@ public class HiveDriver extends AbstractLensDriver { if (opHandle != null) { log.info("CloseQuery hiveHandle: {}", opHandle); try { + fetchLogs(opHandle); getClient().closeOperation(opHandle); } catch (HiveSQLException e) { checkInvalidOperation(handle, e); http://git-wip-us.apache.org/repos/asf/lens/blob/d6b12169/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveInMemoryResultSet.java ---------------------------------------------------------------------- diff --git a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveInMemoryResultSet.java b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveInMemoryResultSet.java index 4d52e22..b39f93c 100644 --- a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveInMemoryResultSet.java +++ b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveInMemoryResultSet.java @@ -112,7 +112,7 @@ public class HiveInMemoryResultSet extends InMemoryResultSet { public boolean hasNext() throws LensException { if (fetchedRowsItr == null || !fetchedRowsItr.hasNext()) { try { - rowSet = client.fetchResults(opHandle, orientation, fetchSize); + rowSet = client.fetchResults(opHandle, orientation, fetchSize, FetchType.QUERY_OUTPUT); orientation = FetchOrientation.FETCH_NEXT; noMoreResults = rowSet.numRows() == 0; if (noMoreResults) { http://git-wip-us.apache.org/repos/asf/lens/blob/d6b12169/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveQueryPlan.java ---------------------------------------------------------------------- diff --git a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveQueryPlan.java b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveQueryPlan.java index f53e2b6..965c5b1 100644 --- a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveQueryPlan.java +++ b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveQueryPlan.java @@ -30,6 +30,7 @@ import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.metadata.Hive; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.metadata.Table; +import org.apache.hadoop.hive.ql.session.SessionState; import lombok.Getter; import lombok.extern.slf4j.Slf4j; @@ -139,7 +140,20 @@ public class HiveQueryPlan extends DriverQueryPlan { } break; case TABLE_SCAN: - // no op + int indentation = getIndentation(explainOutput.get(i)); + i++; + for (; i < explainOutput.size(); i++) { + if (explainOutput.get(i).trim().startsWith("alias: ")) { + String tableName = explainOutput.get(i).trim().substring(7); + if (!tableName.contains(".") && SessionState.get() != null) { + tableName = SessionState.get().getCurrentDatabase() + "." + tableName; + } + addTableToTablesQueried(tableName, metastore); + } + if (i + 1 < explainOutput.size() && getIndentation(explainOutput.get(i+1)) <= indentation) { + break; + } + } break; case PARTITION: String partConditionStr = null; @@ -162,22 +176,7 @@ public class HiveQueryPlan extends DriverQueryPlan { if (explainOutput.get(i).trim().startsWith("name:")) { String table = explainOutput.get(i).trim().substring("name:".length()).trim(); // update tables queried and weights - if (!tablesQueried.contains(table)) { - Table tbl = metastore.getTable(table, false); - if (tbl == null) { - // table not found, possible case if query is create table - log.info("Table {} not found while extracting plan details", table); - continue; - } - tablesQueried.add(table); - String costStr = tbl.getParameters().get(LensConfConstants.STORAGE_COST); - - Double weight = 1d; - if (costStr != null) { - weight = Double.parseDouble(costStr); - } - tableWeights.put(table, weight); - } + addTableToTablesQueried(table, metastore); if (partConditionStr != null) { Set<String> tablePartitions = (Set<String>) partitions.get(table); @@ -201,6 +200,34 @@ public class HiveQueryPlan extends DriverQueryPlan { } } + private void addTableToTablesQueried(String table, Hive metastore) throws HiveException { + if (!tablesQueried.contains(table)) { + Table tbl = metastore.getTable(table, false); + if (tbl == null) { + // table not found, possible case if query is create table + log.info("Table {} not found while extracting plan details", table); + return; + } + tablesQueried.add(table); + String costStr = tbl.getParameters().get(LensConfConstants.STORAGE_COST); + + Double weight = 1d; + if (costStr != null) { + weight = Double.parseDouble(costStr); + } + tableWeights.put(table, weight); + } + } + + private int getIndentation(String s) { + for(int i = 0; i < s.length(); i++) { + if (s.charAt(i) != ' ') { + return i - 1; + } + } + return s.length(); + } + /** * Next state. * @@ -227,7 +254,7 @@ public class HiveQueryPlan extends DriverQueryPlan { return ParserState.GROUPBY_EXPRS; } else if (tr.startsWith("keys:") && state == ParserState.GROUPBY_EXPRS) { return ParserState.GROUPBY_KEYS; - } else if (tr.equals("Path -> Partition:")) { + } else if (tr.equals("Path -> Partition:") || tr.equals("Partition Description:")) { return ParserState.PARTITION_LIST; } else if (tr.equals("Partition") && state == ParserState.PARTITION_LIST) { return ParserState.PARTITION; http://git-wip-us.apache.org/repos/asf/lens/blob/d6b12169/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/RemoteThriftConnection.java ---------------------------------------------------------------------- diff --git a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/RemoteThriftConnection.java b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/RemoteThriftConnection.java index 741294c..54885f7 100644 --- a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/RemoteThriftConnection.java +++ b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/RemoteThriftConnection.java @@ -37,9 +37,10 @@ public class RemoteThriftConnection implements ThriftConnection { private boolean connected; /** The hs2 client. */ - private CLIServiceClient hs2Client; + private RetryingThriftCLIServiceClient.CLIServiceClientWrapper hs2Client; private HiveConf conf; + /** * Instantiates a new remote thrift connection. */ @@ -86,8 +87,8 @@ public class RemoteThriftConnection implements ThriftConnection { @Override public void close() { connected = false; - if (hs2Client instanceof RetryingThriftCLIServiceClient.CLIServiceClientWrapper) { - ((RetryingThriftCLIServiceClient.CLIServiceClientWrapper) hs2Client).closeTransport(); + if (hs2Client != null) { + hs2Client.closeTransport(); } } }
