Repository: incubator-atlas Updated Branches: refs/heads/master f3ac2c0f1 -> dbe106153
ATLAS-448 Hive IllegalArgumentException with Atlas hook enabled on SHOW TRANSACTIONS AND SHOW COMPACTIONS (shwethags) Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/dbe10615 Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/dbe10615 Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/dbe10615 Branch: refs/heads/master Commit: dbe1061538abdd50462cc04f6606d7b5942f27f6 Parents: f3ac2c0 Author: Shwetha GS <[email protected]> Authored: Fri Jan 22 12:18:06 2016 +0530 Committer: Shwetha GS <[email protected]> Committed: Fri Jan 22 12:18:06 2016 +0530 ---------------------------------------------------------------------- .../org/apache/atlas/hive/hook/HiveHook.java | 15 ++++++++- .../org/apache/atlas/hive/hook/HiveHookIT.java | 35 ++++++++++---------- .../src/test/resources/hive-site.xml | 10 ++++++ release-log.txt | 1 + 4 files changed, 43 insertions(+), 18 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/dbe10615/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java ---------------------------------------------------------------------- diff --git a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java index 2cc37c0..5cc4743 100755 --- a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java +++ b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java @@ -49,7 +49,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.concurrent.ExecutorService; import java.util.concurrent.LinkedBlockingQueue; @@ -72,6 +74,8 @@ public class HiveHook extends AtlasHook implements ExecuteWithHookContext { public static final String HOOK_NUM_RETRIES = CONF_PREFIX + "numRetries"; + private static final Map<String, HiveOperation> OPERATION_MAP = new HashMap<>(); + // wait time determines how long we wait before we exit the jvm on // shutdown. Pending requests after that will not be sent. private static final int WAIT_TIME = 3; @@ -131,6 +135,8 @@ public class HiveHook extends AtlasHook implements ExecuteWithHookContext { // shutdown client } }); + + setupOperationMap(); } catch (Exception e) { LOG.info("Attempting to send msg while shutdown in progress.", e); } @@ -140,6 +146,13 @@ public class HiveHook extends AtlasHook implements ExecuteWithHookContext { LOG.info("Created Atlas Hook"); } + private static void setupOperationMap() { + //Populate OPERATION_MAP - string to HiveOperation mapping + for (HiveOperation hiveOperation : HiveOperation.values()) { + OPERATION_MAP.put(hiveOperation.getOperationName(), hiveOperation); + } + } + @Override protected String getNumberOfRetriesPropertyKey() { return HOOK_NUM_RETRIES; @@ -156,7 +169,7 @@ public class HiveHook extends AtlasHook implements ExecuteWithHookContext { event.user = hookContext.getUserName() == null ? hookContext.getUgi().getUserName() : hookContext.getUserName(); event.ugi = hookContext.getUgi(); - event.operation = HiveOperation.valueOf(hookContext.getOperationName()); + event.operation = OPERATION_MAP.get(hookContext.getOperationName()); event.hookType = hookContext.getHookType(); event.queryId = hookContext.getQueryPlan().getQueryId(); event.queryStr = hookContext.getQueryPlan().getQueryStr(); http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/dbe10615/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java ---------------------------------------------------------------------- diff --git a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java index 1c3d9a4..428df52 100755 --- a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java +++ b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java @@ -20,11 +20,11 @@ package org.apache.atlas.hive.hook; import org.apache.atlas.ApplicationProperties; import org.apache.atlas.AtlasClient; -import org.apache.atlas.utils.ParamChecker; import org.apache.atlas.hive.bridge.HiveMetaStoreBridge; import org.apache.atlas.hive.model.HiveDataModelGenerator; import org.apache.atlas.hive.model.HiveDataTypes; import org.apache.atlas.typesystem.Referenceable; +import org.apache.atlas.utils.ParamChecker; import org.apache.commons.configuration.Configuration; import org.apache.commons.lang.RandomStringUtils; import org.apache.commons.lang.StringEscapeUtils; @@ -32,6 +32,7 @@ import org.apache.commons.lang.StringUtils; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.TableType; import org.apache.hadoop.hive.ql.Driver; +import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse; import org.apache.hadoop.hive.ql.session.SessionState; import org.codehaus.jettison.json.JSONArray; import org.codehaus.jettison.json.JSONObject; @@ -43,6 +44,8 @@ import org.testng.annotations.Test; import java.io.File; import java.util.Map; +import static org.testng.Assert.assertEquals; + public class HiveHookIT { public static final Logger LOG = org.slf4j.LoggerFactory.getLogger(HiveHookIT.class); @@ -67,8 +70,10 @@ public class HiveHookIT { } private void runCommand(String cmd) throws Exception { + LOG.debug("Running command '{}'", cmd); ss.setCommandType(null); - driver.run(cmd); + CommandProcessorResponse response = driver.run(cmd); + assertEquals(response.getResponseCode(), 0); } @Test @@ -149,17 +154,6 @@ public class HiveHookIT { assertDatabaseIsRegistered(DEFAULT_DB); } - @Test - public void testRenameTable() throws Exception { - String tableName = createTable(); - String newTableName = tableName(); - runCommand(String.format("alter table %s rename to %s", tableName, newTableName)); - - assertTableIsRegistered(DEFAULT_DB, newTableName); - assertTableIsNotRegistered(DEFAULT_DB, tableName); - } - - private String assertColumnIsRegistered(String colName) throws Exception { LOG.debug("Searching for column {}", colName); String query = @@ -265,8 +259,8 @@ public class HiveHookIT { assertProcessIsRegistered(query); } - @Test(enabled = false) - public void testAlterTable() throws Exception { + @Test + public void testAlterTableRename() throws Exception { String tableName = createTable(); String newName = tableName(); String query = "alter table " + tableName + " rename to " + newName; @@ -276,8 +270,8 @@ public class HiveHookIT { assertTableIsNotRegistered(DEFAULT_DB, tableName); } - @Test(enabled = false) - public void testAlterView() throws Exception { + @Test + public void testAlterViewRename() throws Exception { String tableName = createTable(); String viewName = tableName(); String newName = tableName(); @@ -396,6 +390,13 @@ public class HiveHookIT { Assert.assertTrue(vertices.has(table2Id)); } + //For ATLAS-448 + @Test + public void testNoopOperation() throws Exception { + runCommand("show compactions"); + runCommand("show transactions"); + } + public interface Predicate { /** http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/dbe10615/addons/hive-bridge/src/test/resources/hive-site.xml ---------------------------------------------------------------------- diff --git a/addons/hive-bridge/src/test/resources/hive-site.xml b/addons/hive-bridge/src/test/resources/hive-site.xml index b106903..f1facb8 100644 --- a/addons/hive-bridge/src/test/resources/hive-site.xml +++ b/addons/hive-bridge/src/test/resources/hive-site.xml @@ -50,4 +50,14 @@ <name>fs.pfile.impl</name> <value>org.apache.hadoop.fs.ProxyLocalFileSystem</value> </property> + + <property> + <name>hive.in.test</name> + <value>true</value> + </property> + + <property> + <name>hive.zookeeper.quorum</name> + <value>localhost:19026</value> + </property> </configuration> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/dbe10615/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index 819300a..66ecf9a 100644 --- a/release-log.txt +++ b/release-log.txt @@ -7,6 +7,7 @@ ATLAS-409 Atlas will not import avro tables with schema read from a file (dosset ATLAS-379 Create sqoop and falcon metadata addons (venkatnrangan,bvellanki,sowmyaramesh via shwethags) ALL CHANGES: +ATLAS-448 Hive IllegalArgumentException with Atlas hook enabled on SHOW TRANSACTIONS AND SHOW COMPACTIONS (shwethags) ATLAS-181 Integrate storm topology metadata into Atlas (svenkat,yhemanth via shwethags) ATLAS-311 UI: Local storage for traits - caching [not cleared on refresh] To be cleared on time lapse for 1hr (Anilg via shwethags) ATLAS-106 Store createTimestamp and modified timestamp separately for an entity (dkantor via shwethags)
