Repository: nifi Updated Branches: refs/heads/master 1dbfcb944 -> c575a9893
NIFI-5095: Suppress SET property parse failure at Hive processors Log debug message when ParseException is thrown. Log warning message if other unknown Exception is thrown. Signed-off-by: Matthew Burgess <[email protected]> This closes #2645 Project: http://git-wip-us.apache.org/repos/asf/nifi/repo Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/c575a989 Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/c575a989 Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/c575a989 Branch: refs/heads/master Commit: c575a98936087dcd360ed7715a9e2341e46c7f0d Parents: 1dbfcb9 Author: Koji Kawamura <[email protected]> Authored: Thu Apr 19 15:45:16 2018 +0900 Committer: Matthew Burgess <[email protected]> Committed: Fri Apr 20 11:24:02 2018 -0400 ---------------------------------------------------------------------- .../hive/AbstractHiveQLProcessor.java | 13 +++- .../nifi/processors/hive/TestHiveParser.java | 67 +++++++++++++++++--- 2 files changed, 70 insertions(+), 10 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/nifi/blob/c575a989/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/main/java/org/apache/nifi/processors/hive/AbstractHiveQLProcessor.java ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/main/java/org/apache/nifi/processors/hive/AbstractHiveQLProcessor.java b/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/main/java/org/apache/nifi/processors/hive/AbstractHiveQLProcessor.java index cd57090..f1ef5fd 100644 --- a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/main/java/org/apache/nifi/processors/hive/AbstractHiveQLProcessor.java +++ b/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/main/java/org/apache/nifi/processors/hive/AbstractHiveQLProcessor.java @@ -40,6 +40,7 @@ import java.sql.SQLException; import java.sql.Time; import java.sql.Timestamp; import java.sql.Types; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.Map; @@ -274,8 +275,16 @@ public abstract class AbstractHiveQLProcessor extends AbstractSessionFactoryProc } } - protected Set<TableName> findTableNames(final String query) throws ParseException { - final ASTNode node = new ParseDriver().parse(normalize(query)); + protected Set<TableName> findTableNames(final String query) { + final ASTNode node; + try { + node = new ParseDriver().parse(normalize(query)); + } catch (ParseException e) { + // If failed to parse the query, just log a message, but continue. + getLogger().debug("Failed to parse query: {} due to {}", new Object[]{query, e}, e); + return Collections.emptySet(); + } + final HashSet<TableName> tableNames = new HashSet<>(); findTableNames(node, tableNames); return tableNames; http://git-wip-us.apache.org/repos/asf/nifi/blob/c575a989/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/test/java/org/apache/nifi/processors/hive/TestHiveParser.java ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/test/java/org/apache/nifi/processors/hive/TestHiveParser.java b/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/test/java/org/apache/nifi/processors/hive/TestHiveParser.java index 50ac0d8..a6f4e36 100644 --- a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/test/java/org/apache/nifi/processors/hive/TestHiveParser.java +++ b/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/test/java/org/apache/nifi/processors/hive/TestHiveParser.java @@ -18,7 +18,11 @@ package org.apache.nifi.processors.hive; import org.apache.nifi.processor.ProcessContext; import org.apache.nifi.processor.ProcessSessionFactory; +import org.apache.nifi.processor.ProcessorInitializationContext; import org.apache.nifi.processor.exception.ProcessException; +import org.apache.nifi.util.MockProcessContext; +import org.apache.nifi.util.MockProcessorInitializationContext; +import org.junit.Before; import org.junit.Test; import java.util.Set; @@ -29,13 +33,20 @@ import static org.junit.Assert.assertTrue; public class TestHiveParser extends AbstractHiveQLProcessor { + @Before + public void initialize() { + final MockProcessContext processContext = new MockProcessContext(this); + final ProcessorInitializationContext initializationContext = new MockProcessorInitializationContext(this, processContext); + initialize(initializationContext); + } + @Override public void onTrigger(ProcessContext context, ProcessSessionFactory sessionFactory) throws ProcessException { } @Test - public void parseSelect() throws Exception { + public void parseSelect() { String query = "select a.empid, to_something(b.saraly) from " + "company.emp a inner join default.salary b where a.empid = b.empid"; final Set<TableName> tableNames = findTableNames(query); @@ -46,7 +57,7 @@ public class TestHiveParser extends AbstractHiveQLProcessor { } @Test - public void parseSelectPrepared() throws Exception { + public void parseSelectPrepared() { String query = "select empid from company.emp a where a.firstName = ?"; final Set<TableName> tableNames = findTableNames(query); System.out.printf("tableNames=%s\n", tableNames); @@ -56,7 +67,7 @@ public class TestHiveParser extends AbstractHiveQLProcessor { @Test - public void parseLongSelect() throws Exception { + public void parseLongSelect() { String query = "select\n" + "\n" + " i_item_id,\n" + @@ -164,7 +175,7 @@ public class TestHiveParser extends AbstractHiveQLProcessor { } @Test - public void parseSelectInsert() throws Exception { + public void parseSelectInsert() { String query = "insert into databaseA.tableA select key, max(value) from databaseA.tableA where category = 'x'"; // The same database.tableName can appear two times for input and output. @@ -183,7 +194,7 @@ public class TestHiveParser extends AbstractHiveQLProcessor { } @Test - public void parseInsert() throws Exception { + public void parseInsert() { String query = "insert into databaseB.tableB1 select something from tableA1 a1 inner join tableA2 a2 where a1.id = a2.id"; final Set<TableName> tableNames = findTableNames(query); @@ -203,7 +214,7 @@ public class TestHiveParser extends AbstractHiveQLProcessor { } @Test - public void parseUpdate() throws Exception { + public void parseUpdate() { String query = "update table_a set y = 'updated' where x > 100"; final Set<TableName> tableNames = findTableNames(query); @@ -213,7 +224,7 @@ public class TestHiveParser extends AbstractHiveQLProcessor { } @Test - public void parseDelete() throws Exception { + public void parseDelete() { String query = "delete from table_a where x > 100"; final Set<TableName> tableNames = findTableNames(query); @@ -223,7 +234,7 @@ public class TestHiveParser extends AbstractHiveQLProcessor { } @Test - public void parseDDL() throws Exception { + public void parseDDL() { String query = "CREATE TABLE IF NOT EXISTS EMPLOYEES(\n" + "EmployeeID INT,FirstName STRING, Title STRING,\n" + "State STRING, Laptop STRING)\n" + @@ -237,5 +248,45 @@ public class TestHiveParser extends AbstractHiveQLProcessor { assertTrue(tableNames.contains(new TableName(null, "EMPLOYEES", false))); } + @Test + public void parseSetProperty() { + String query = " set 'hive.exec.dynamic.partition.mode'=nonstrict"; + final Set<TableName> tableNames = findTableNames(query); + System.out.printf("tableNames=%s\n", tableNames); + assertEquals(0, tableNames.size()); + } + + @Test + public void parseSetRole() { + String query = "set role all"; + final Set<TableName> tableNames = findTableNames(query); + System.out.printf("tableNames=%s\n", tableNames); + assertEquals(0, tableNames.size()); + } + + @Test + public void parseShowRoles() { + String query = "show roles"; + final Set<TableName> tableNames = findTableNames(query); + System.out.printf("tableNames=%s\n", tableNames); + assertEquals(0, tableNames.size()); + } + + @Test + public void parseMsck() { + String query = "msck repair table table_a"; + final Set<TableName> tableNames = findTableNames(query); + System.out.printf("tableNames=%s\n", tableNames); + assertEquals(1, tableNames.size()); + assertTrue(tableNames.contains(new TableName(null, "table_a", false))); + } + + @Test + public void parseAddJar() { + String query = "ADD JAR hdfs:///tmp/my_jar.jar"; + final Set<TableName> tableNames = findTableNames(query); + System.out.printf("tableNames=%s\n", tableNames); + assertEquals(0, tableNames.size()); + } }
