Sentry-1321: Implement HMSFollower in Sentry service which reads the NotificationLog entries (Sravya Tirukkovalur, Reviewed by: Hao Hao) (Append)
Change-Id: If6e3d4a754b71cc0ba0cec1e34a44e6d9816ea38 Project: http://git-wip-us.apache.org/repos/asf/sentry/repo Commit: http://git-wip-us.apache.org/repos/asf/sentry/commit/908072d6 Tree: http://git-wip-us.apache.org/repos/asf/sentry/tree/908072d6 Diff: http://git-wip-us.apache.org/repos/asf/sentry/diff/908072d6 Branch: refs/heads/sentry-ha-redesign Commit: 908072d66bbabd91491a7d8fe8d9fa558dbc9c46 Parents: 2d5ed99 Author: Sravya Tirukkovalur <[email protected]> Authored: Mon Aug 15 17:25:47 2016 -0700 Committer: Sravya Tirukkovalur <[email protected]> Committed: Mon Aug 15 17:25:47 2016 -0700 ---------------------------------------------------------------------- sentry-binding/sentry-binding-hive-conf/pom.xml | 62 +++ .../sentry/binding/hive/conf/HiveAuthzConf.java | 269 ++++++++++++ .../sentry-binding-hive-follower/pom.xml | 71 ++++ .../json/JSONAlterPartitionMessage.java | 78 ++++ .../messaging/json/JSONAlterTableMessage.java | 68 +++ .../json/SentryJSONAddPartitionMessage.java | 49 +++ .../json/SentryJSONAlterPartitionMessage.java | 53 +++ .../json/SentryJSONAlterTableMessage.java | 50 +++ .../json/SentryJSONCreateDatabaseMessage.java | 44 ++ .../json/SentryJSONCreateTableMessage.java | 45 ++ .../json/SentryJSONDropDatabaseMessage.java | 44 ++ .../json/SentryJSONDropPartitionMessage.java | 49 +++ .../json/SentryJSONDropTableMessage.java | 45 ++ .../json/SentryJSONMessageDeserializer.java | 110 +++++ .../json/SentryJSONMessageFactory.java | 177 ++++++++ .../SentryInvalidHMSEventException.java | 31 ++ .../exception/SentryStandbyException.java | 4 + .../SentryPolicyServiceClientDefaultImpl.java | 12 +- .../sentry/service/thrift/HMSFollower.java | 411 +++++++++++++++++++ .../thrift/PoolClientInvocationHandler.java | 305 +++++++++++--- .../thrift/SentryServiceClientFactory.java | 11 +- .../thrift/SentryServiceClientPoolFactory.java | 11 +- .../thrift/TestSentryServiceFailureCase.java | 5 +- .../TestSentryServiceWithInvalidMsgSize.java | 2 +- .../sentry/service/thrift/TestHMSFollower.java | 165 ++++++++ .../thrift/TestPoolClientInvocationHandler.java | 69 ++++ 26 files changed, 2173 insertions(+), 67 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sentry/blob/908072d6/sentry-binding/sentry-binding-hive-conf/pom.xml ---------------------------------------------------------------------- diff --git a/sentry-binding/sentry-binding-hive-conf/pom.xml b/sentry-binding/sentry-binding-hive-conf/pom.xml new file mode 100644 index 0000000..2cae541 --- /dev/null +++ b/sentry-binding/sentry-binding-hive-conf/pom.xml @@ -0,0 +1,62 @@ +<?xml version="1.0"?> +<!-- +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +--> +<project xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"; xmlns="http://maven.apache.org/POM/4.0.0"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";> + <modelVersion>4.0.0</modelVersion> + + <parent> + <groupId>org.apache.sentry</groupId> + <artifactId>sentry-binding</artifactId> + <version>1.8.0-SNAPSHOT</version> + </parent> + + <artifactId>sentry-binding-hive-conf</artifactId> + <name>Sentry Hive Binding Configuration</name> + + <dependencies> + <dependency> + <groupId>org.apache.thrift</groupId> + <artifactId>libthrift</artifactId> + </dependency> + <dependency> + <groupId>org.apache.derby</groupId> + <artifactId>derby</artifactId> + </dependency> + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.apache.hive</groupId> + <artifactId>hive-exec</artifactId> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.apache.hive</groupId> + <artifactId>hive-service</artifactId> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-common</artifactId> + <scope>provided</scope> + </dependency> + </dependencies> + +</project> http://git-wip-us.apache.org/repos/asf/sentry/blob/908072d6/sentry-binding/sentry-binding-hive-conf/src/main/java/org/apache/sentry/binding/hive/conf/HiveAuthzConf.java ---------------------------------------------------------------------- diff --git a/sentry-binding/sentry-binding-hive-conf/src/main/java/org/apache/sentry/binding/hive/conf/HiveAuthzConf.java b/sentry-binding/sentry-binding-hive-conf/src/main/java/org/apache/sentry/binding/hive/conf/HiveAuthzConf.java new file mode 100644 index 0000000..4de755f --- /dev/null +++ b/sentry-binding/sentry-binding-hive-conf/src/main/java/org/apache/sentry/binding/hive/conf/HiveAuthzConf.java @@ -0,0 +1,269 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sentry.binding.hive.conf; + +import java.net.MalformedURLException; +import java.net.URL; +import java.util.HashMap; +import java.util.Map; +import java.util.Map.Entry; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.conf.HiveConf; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + + +public class HiveAuthzConf extends Configuration { + + /** + * Configuration key used in hive-site.xml to point at sentry-site.xml + */ + public static final String HIVE_ACCESS_CONF_URL = "hive.access.conf.url"; + public static final String HIVE_SENTRY_CONF_URL = "hive.sentry.conf.url"; + public static final String HIVE_ACCESS_SUBJECT_NAME = "hive.access.subject.name"; + public static final String HIVE_SENTRY_SUBJECT_NAME = "hive.sentry.subject.name"; + public static final String HIVE_SENTRY_AUTH_ERRORS = "sentry.hive.authorization.errors"; + public static final String HIVE_SENTRY_MOCK_COMPILATION = "sentry.hive.mock.compilation"; + public static final String HIVE_SENTRY_MOCK_ERROR = "sentry.hive.mock.error"; + public static final String HIVE_SENTRY_PRIVILEGE_ERROR_MESSAGE = "No valid privileges"; + /** + * Property used to persist the role set in the session. This is not public for now. + */ + public static final String SENTRY_ACTIVE_ROLE_SET = "hive.sentry.active.role.set"; + + public static final String HIVE_SENTRY_SECURITY_COMMAND_WHITELIST = + "hive.sentry.security.command.whitelist"; + public static final String HIVE_SENTRY_SECURITY_COMMAND_WHITELIST_DEFAULT = + "set,reset,reload"; + + public static final String HIVE_SENTRY_SERDE_WHITELIST = "hive.sentry.serde.whitelist"; + public static final String HIVE_SENTRY_SERDE_WHITELIST_DEFAULT = "org.apache.hadoop.hive.serde2"; + + // Disable the serde Uri privileges by default for backward compatibilities. + public static final String HIVE_SENTRY_SERDE_URI_PRIVILIEGES_ENABLED = "hive.sentry.turn.on.serde.uri.privileges"; + public static final boolean HIVE_SENTRY_SERDE_URI_PRIVILIEGES_ENABLED_DEFAULT = false; + + public static final String HIVE_UDF_WHITE_LIST = + "concat,substr,substring,space,repeat,ascii,lpad,rpad,size,round,floor,sqrt,ceil," + + "ceiling,rand,abs,pmod,ln,log2,sin,asin,cos,acos,log10,log,exp,power,pow,sign,pi," + + "degrees,radians,atan,tan,e,conv,bin,hex,unhex,base64,unbase64,encode,decode,upper," + + "lower,ucase,lcase,trim,ltrim,rtrim,length,reverse,field,find_in_set,initcap,like," + + "rlike,regexp,regexp_replace,regexp_extract,parse_url,nvl,split,str_to_map,translate" + + ",positive,negative,day,dayofmonth,month,year,hour,minute,second,from_unixtime," + + "to_date,weekofyear,last_day,date_add,date_sub,datediff,add_months,get_json_object," + + "xpath_string,xpath_boolean,xpath_number,xpath_double,xpath_float,xpath_long," + + "xpath_int,xpath_short,xpath,+,-,*,/,%,div,&,|,^,~,current_database,isnull," + + "isnotnull,if,in,and,or,=,==,<=>,!=,<>,<,<=,>,>=,not,!,between,ewah_bitmap_and," + + "ewah_bitmap_or,ewah_bitmap_empty,boolean,tinyint,smallint,int,bigint,float,double," + + "string,date,timestamp,binary,decimal,varchar,char,max,min,sum,count,avg,std,stddev," + + "stddev_pop,stddev_samp,variance,var_pop,var_samp,covar_pop,covar_samp,corr," + + "histogram_numeric,percentile_approx,collect_set,collect_list,ngrams," + + "context_ngrams,ewah_bitmap,compute_stats,percentile," + + "array,assert_true,map,struct,named_struct,create_union,case,when,hash,coalesce," + + "index,in_file,instr,locate,elt,concat_ws,sort_array," + + "array_contains,sentences,map_keys,map_values,format_number,printf,greatest,least," + + "from_utc_timestamp,to_utc_timestamp,unix_timestamp,to_unix_timestamp,explode," + + "inline,json_tuple,parse_url_tuple,posexplode,stack,lead,lag,row_number,rank," + + "dense_rank,percent_rank,cume_dist,ntile,first_value,last_value,noop,noopwithmap," + + "noopstreaming,noopwithmapstreaming,windowingtablefunction,matchpath"; + + public static final String HIVE_UDF_BLACK_LIST = "reflect,reflect2,java_method"; + + /** + * Config setting definitions + */ + public static enum AuthzConfVars { + AUTHZ_PROVIDER("sentry.provider", + "org.apache.sentry.provider.common.HadoopGroupResourceAuthorizationProvider"), + AUTHZ_PROVIDER_RESOURCE("sentry.hive.provider.resource", ""), + AUTHZ_PROVIDER_BACKEND("sentry.hive.provider.backend", "org.apache.sentry.provider.file.SimpleFileProviderBackend"), + AUTHZ_POLICY_ENGINE("sentry.hive.policy.engine", "org.apache.sentry.policy.engine.common.CommonPolicyEngine"), + AUTHZ_POLICY_FILE_FORMATTER( + "sentry.hive.policy.file.formatter", + "org.apache.sentry.binding.hive.SentryIniPolicyFileFormatter"), + AUTHZ_SERVER_NAME("sentry.hive.server", ""), + AUTHZ_RESTRICT_DEFAULT_DB("sentry.hive.restrict.defaultDB", "false"), + SENTRY_TESTING_MODE("sentry.hive.testing.mode", "false"), + AUTHZ_ALLOW_HIVE_IMPERSONATION("sentry.hive.allow.hive.impersonation", "false"), + AUTHZ_ONFAILURE_HOOKS("sentry.hive.failure.hooks", ""), + AUTHZ_METASTORE_SERVICE_USERS("sentry.metastore.service.users", null), + AUTHZ_SYNC_ALTER_WITH_POLICY_STORE("sentry.hive.sync.alter", "true"), + AUTHZ_SYNC_CREATE_WITH_POLICY_STORE("sentry.hive.sync.create", "false"), + AUTHZ_SYNC_DROP_WITH_POLICY_STORE("sentry.hive.sync.drop", "true"), + + AUTHZ_PROVIDER_DEPRECATED("hive.sentry.provider", + "org.apache.sentry.provider.file.ResourceAuthorizationProvider"), + AUTHZ_PROVIDER_RESOURCE_DEPRECATED("hive.sentry.provider.resource", ""), + AUTHZ_SERVER_NAME_DEPRECATED("hive.sentry.server", ""), + AUTHZ_RESTRICT_DEFAULT_DB_DEPRECATED("hive.sentry.restrict.defaultDB", "false"), + SENTRY_TESTING_MODE_DEPRECATED("hive.sentry.testing.mode", "false"), + AUTHZ_ALLOW_HIVE_IMPERSONATION_DEPRECATED("hive.sentry.allow.hive.impersonation", "false"), + AUTHZ_ONFAILURE_HOOKS_DEPRECATED("hive.sentry.failure.hooks", ""); + + private final String varName; + private final String defaultVal; + + AuthzConfVars(String varName, String defaultVal) { + this.varName = varName; + this.defaultVal = defaultVal; + } + + public String getVar() { + return varName; + } + + public String getDefault() { + return defaultVal; + } + + public static String getDefault(String varName) { + for (AuthzConfVars oneVar : AuthzConfVars.values()) { + if(oneVar.getVar().equalsIgnoreCase(varName)) { + return oneVar.getDefault(); + } + } + return null; + } + } + + // map of current property names - > deprecated property names. + // The binding layer code should work if the deprecated property names are provided, + // as long as the new property names aren't also provided. Since the binding code + // only calls the new property names, we require a map from current names to deprecated + // names in order to check if the deprecated name of a property was set. + private static final Map<String, AuthzConfVars> currentToDeprecatedProps = + new HashMap<String, AuthzConfVars>(); + static { + currentToDeprecatedProps.put(AuthzConfVars.AUTHZ_PROVIDER.getVar(), AuthzConfVars.AUTHZ_PROVIDER_DEPRECATED); + currentToDeprecatedProps.put(AuthzConfVars.AUTHZ_PROVIDER_RESOURCE.getVar(), AuthzConfVars.AUTHZ_PROVIDER_RESOURCE_DEPRECATED); + currentToDeprecatedProps.put(AuthzConfVars.AUTHZ_SERVER_NAME.getVar(), AuthzConfVars.AUTHZ_SERVER_NAME_DEPRECATED); + currentToDeprecatedProps.put(AuthzConfVars.AUTHZ_RESTRICT_DEFAULT_DB.getVar(), AuthzConfVars.AUTHZ_RESTRICT_DEFAULT_DB_DEPRECATED); + currentToDeprecatedProps.put(AuthzConfVars.SENTRY_TESTING_MODE.getVar(), AuthzConfVars.SENTRY_TESTING_MODE_DEPRECATED); + currentToDeprecatedProps.put(AuthzConfVars.AUTHZ_ALLOW_HIVE_IMPERSONATION.getVar(), AuthzConfVars.AUTHZ_ALLOW_HIVE_IMPERSONATION_DEPRECATED); + currentToDeprecatedProps.put(AuthzConfVars.AUTHZ_ONFAILURE_HOOKS.getVar(), AuthzConfVars.AUTHZ_ONFAILURE_HOOKS_DEPRECATED); + }; + + private static final Logger LOG = LoggerFactory + .getLogger(HiveAuthzConf.class); + public static final String AUTHZ_SITE_FILE = "sentry-site.xml"; + private final String hiveAuthzSiteFile; + + public HiveAuthzConf(URL hiveAuthzSiteURL) { + super(); + LOG.info("DefaultFS: " + super.get("fs.defaultFS")); + addResource(hiveAuthzSiteURL); + applySystemProperties(); + LOG.info("DefaultFS: " + super.get("fs.defaultFS")); + this.hiveAuthzSiteFile = hiveAuthzSiteURL.toString(); + } + /** + * Apply system properties to this object if the property name is defined in ConfVars + * and the value is non-null and not an empty string. + */ + private void applySystemProperties() { + Map<String, String> systemProperties = getConfSystemProperties(); + for (Entry<String, String> systemProperty : systemProperties.entrySet()) { + this.set(systemProperty.getKey(), systemProperty.getValue()); + } + } + + /** + * This method returns a mapping from config variable name to its value for all config variables + * which have been set using System properties + */ + public static Map<String, String> getConfSystemProperties() { + Map<String, String> systemProperties = new HashMap<String, String>(); + + for (AuthzConfVars oneVar : AuthzConfVars.values()) { + String value = System.getProperty(oneVar.getVar()); + if (value != null && value.length() > 0) { + systemProperties.put(oneVar.getVar(), value); + } + } + return systemProperties; + } + + @Override + public String get(String varName) { + return get(varName, null); + } + + @Override + public String get(String varName, String defaultVal) { + String retVal = super.get(varName); + if (retVal == null) { + // check if the deprecated value is set here + if (currentToDeprecatedProps.containsKey(varName)) { + retVal = super.get(currentToDeprecatedProps.get(varName).getVar()); + } + if (retVal == null) { + retVal = AuthzConfVars.getDefault(varName); + } else { + LOG.warn("Using the deprecated config setting " + currentToDeprecatedProps.get(varName).getVar() + + " instead of " + varName); + } + } + if (retVal == null) { + retVal = defaultVal; + } + return retVal; + } + + public String getHiveAuthzSiteFile() { + return hiveAuthzSiteFile; + } + + /** + * Extract the authz config file path from given hive conf and load the authz config + * @param hiveConf + * @return + * @throws IllegalArgumentException + */ + public static HiveAuthzConf getAuthzConf(HiveConf hiveConf) + throws IllegalArgumentException { + boolean depreicatedConfigFile = false; + + String hiveAuthzConf = hiveConf.get(HiveAuthzConf.HIVE_SENTRY_CONF_URL); + if (hiveAuthzConf == null + || (hiveAuthzConf = hiveAuthzConf.trim()).isEmpty()) { + hiveAuthzConf = hiveConf.get(HiveAuthzConf.HIVE_ACCESS_CONF_URL); + depreicatedConfigFile = true; + } + + if (hiveAuthzConf == null + || (hiveAuthzConf = hiveAuthzConf.trim()).isEmpty()) { + throw new IllegalArgumentException("Configuration key " + + HiveAuthzConf.HIVE_SENTRY_CONF_URL + " value '" + hiveAuthzConf + + "' is invalid."); + } + + try { + return new HiveAuthzConf(new URL(hiveAuthzConf)); + } catch (MalformedURLException e) { + if (depreicatedConfigFile) { + throw new IllegalArgumentException("Configuration key " + + HiveAuthzConf.HIVE_ACCESS_CONF_URL + + " specifies a malformed URL '" + hiveAuthzConf + "'", e); + } else { + throw new IllegalArgumentException("Configuration key " + + HiveAuthzConf.HIVE_SENTRY_CONF_URL + + " specifies a malformed URL '" + hiveAuthzConf + "'", e); + } + } + } +} http://git-wip-us.apache.org/repos/asf/sentry/blob/908072d6/sentry-binding/sentry-binding-hive-follower/pom.xml ---------------------------------------------------------------------- diff --git a/sentry-binding/sentry-binding-hive-follower/pom.xml b/sentry-binding/sentry-binding-hive-follower/pom.xml new file mode 100644 index 0000000..bcfb417 --- /dev/null +++ b/sentry-binding/sentry-binding-hive-follower/pom.xml @@ -0,0 +1,71 @@ +<?xml version="1.0"?> +<!-- +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +--> +<project xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"; xmlns="http://maven.apache.org/POM/4.0.0"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";> + <modelVersion>4.0.0</modelVersion> + + <parent> + <groupId>org.apache.sentry</groupId> + <artifactId>sentry-binding</artifactId> + <version>1.8.0-SNAPSHOT</version> + </parent> + + <artifactId>sentry-binding-hive-follower</artifactId> + <name>Hive follower for Sentry</name> + + <properties> + <datanucleus-api-jdo.version>3.2.6</datanucleus-api-jdo.version> + <datanucleus-core.version>3.2.12</datanucleus-core.version> + <datanucleus-rdbms.version>3.2.12</datanucleus-rdbms.version> + </properties> + + <dependencies> + <dependency> + <groupId>org.datanucleus</groupId> + <artifactId>datanucleus-core</artifactId> + <version>${datanucleus-core.version}</version> + </dependency> + <dependency> + <groupId>org.datanucleus</groupId> + <artifactId>datanucleus-api-jdo</artifactId> + <version>${datanucleus-api-jdo.version}</version> + </dependency> + <dependency> + <groupId>org.datanucleus</groupId> + <artifactId>datanucleus-rdbms</artifactId> + <version>${datanucleus-rdbms.version}</version> + </dependency> + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.apache.hive</groupId> + <artifactId>hive-exec</artifactId> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.apache.hive.hcatalog</groupId> + <artifactId>hive-hcatalog-server-extensions</artifactId> + <version>${hive.version}</version> + <scope>compile</scope> + </dependency> + </dependencies> + +</project> http://git-wip-us.apache.org/repos/asf/sentry/blob/908072d6/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/JSONAlterPartitionMessage.java ---------------------------------------------------------------------- diff --git a/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/JSONAlterPartitionMessage.java b/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/JSONAlterPartitionMessage.java new file mode 100644 index 0000000..890186b --- /dev/null +++ b/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/JSONAlterPartitionMessage.java @@ -0,0 +1,78 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.sentry.binding.metastore.messaging.json; + +import org.apache.hive.hcatalog.messaging.AlterPartitionMessage; +import org.codehaus.jackson.annotate.JsonProperty; + +import java.util.List; + +/* +* This is only needed as corresponding class in Hive 1.1.0 does not have a default constructor + */ +public class JSONAlterPartitionMessage extends AlterPartitionMessage { + @JsonProperty + String server; + @JsonProperty + String servicePrincipal; + @JsonProperty + String db; + @JsonProperty + String table; + @JsonProperty + Long timestamp; + @JsonProperty + List<String> values; + + public JSONAlterPartitionMessage() {} + public JSONAlterPartitionMessage(String server, String servicePrincipal, String db, String table, List<String> values, Long timestamp) { + this.server = server; + this.servicePrincipal = servicePrincipal; + this.db = db; + this.table = table; + this.timestamp = timestamp; + this.values = values; + this.checkValid(); + } + + public String getServer() { + return this.server; + } + + public String getServicePrincipal() { + return this.servicePrincipal; + } + + public String getDB() { + return this.db; + } + + public Long getTimestamp() { + return this.timestamp; + } + + public String getTable() { + return this.table; + } + + public List<String> getValues() { + return this.values; + } + +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/sentry/blob/908072d6/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/JSONAlterTableMessage.java ---------------------------------------------------------------------- diff --git a/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/JSONAlterTableMessage.java b/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/JSONAlterTableMessage.java new file mode 100644 index 0000000..76211c3 --- /dev/null +++ b/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/JSONAlterTableMessage.java @@ -0,0 +1,68 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.sentry.binding.metastore.messaging.json; + +import org.apache.hive.hcatalog.messaging.AlterTableMessage; +import org.codehaus.jackson.annotate.JsonProperty; + +/** + * This class is required as this class does not have a default contructor in Hive 1.1.0 + */ +public class JSONAlterTableMessage extends AlterTableMessage { + @JsonProperty + String server; + @JsonProperty + String servicePrincipal; + @JsonProperty + String db; + @JsonProperty + String table; + @JsonProperty + Long timestamp; + + public JSONAlterTableMessage() {} + public JSONAlterTableMessage(String server, String servicePrincipal, String db, String table, Long timestamp) { + this.server = server; + this.servicePrincipal = servicePrincipal; + this.db = db; + this.table = table; + this.timestamp = timestamp; + this.checkValid(); + } + + public String getServer() { + return this.server; + } + + public String getServicePrincipal() { + return this.servicePrincipal; + } + + public String getDB() { + return this.db; + } + + public Long getTimestamp() { + return this.timestamp; + } + + public String getTable() { + return this.table; + } +} http://git-wip-us.apache.org/repos/asf/sentry/blob/908072d6/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONAddPartitionMessage.java ---------------------------------------------------------------------- diff --git a/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONAddPartitionMessage.java b/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONAddPartitionMessage.java new file mode 100644 index 0000000..c0c469c --- /dev/null +++ b/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONAddPartitionMessage.java @@ -0,0 +1,49 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.sentry.binding.metastore.messaging.json; + +import org.apache.hive.hcatalog.messaging.json.JSONAddPartitionMessage; +import org.codehaus.jackson.annotate.JsonProperty; + +import java.util.List; +import java.util.Map; + +public class SentryJSONAddPartitionMessage extends JSONAddPartitionMessage { + @JsonProperty + List<String> locations; + + public SentryJSONAddPartitionMessage() { + } + + public SentryJSONAddPartitionMessage(String server, String servicePrincipal, String db, String table, + List<Map<String, String>> partitions, Long timestamp, List<String> locations) { + super(server, servicePrincipal, db, table, partitions, timestamp); + this.locations = locations; + } + + public List<String> getLocations() { + return locations; + } + + @Override + public String toString() { + return SentryJSONMessageDeserializer.serialize(this); + } + +} http://git-wip-us.apache.org/repos/asf/sentry/blob/908072d6/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONAlterPartitionMessage.java ---------------------------------------------------------------------- diff --git a/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONAlterPartitionMessage.java b/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONAlterPartitionMessage.java new file mode 100644 index 0000000..99eb67a --- /dev/null +++ b/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONAlterPartitionMessage.java @@ -0,0 +1,53 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.sentry.binding.metastore.messaging.json; + +import org.codehaus.jackson.annotate.JsonProperty; + +import java.util.List; + +public class SentryJSONAlterPartitionMessage extends JSONAlterPartitionMessage{ + @JsonProperty + String location; + @JsonProperty + String oldLocation; + + public SentryJSONAlterPartitionMessage() { + } + + public SentryJSONAlterPartitionMessage(String server, String servicePrincipal, String db, String table, + List<String> values, Long timestamp, String oldlocation, String newLocation) { + super(server, servicePrincipal, db, table, values, timestamp); + this.location = newLocation; + this.oldLocation = oldlocation; + } + + public String getLocation() { + return location; + } + + public String getOldLocation() { + return oldLocation; + } + + @Override + public String toString() { + return SentryJSONMessageDeserializer.serialize(this); + } +} http://git-wip-us.apache.org/repos/asf/sentry/blob/908072d6/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONAlterTableMessage.java ---------------------------------------------------------------------- diff --git a/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONAlterTableMessage.java b/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONAlterTableMessage.java new file mode 100644 index 0000000..6e59e25 --- /dev/null +++ b/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONAlterTableMessage.java @@ -0,0 +1,50 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.sentry.binding.metastore.messaging.json; + +import org.codehaus.jackson.annotate.JsonProperty; + +public class SentryJSONAlterTableMessage extends JSONAlterTableMessage { + @JsonProperty + String location; //newLocation + @JsonProperty + String oldLocation; + + public SentryJSONAlterTableMessage() { + } + + public SentryJSONAlterTableMessage(String server, String servicePrincipal, String db, String table, + Long timestamp, String oldLocation, String location) { + super(server, servicePrincipal, db, table, timestamp); + this.location = location; + this.oldLocation = oldLocation; + } + + public String getLocation() { + return location; + } + public String getOldLocation() { + return oldLocation; + } + + @Override + public String toString() { + return SentryJSONMessageDeserializer.serialize(this); + } +} http://git-wip-us.apache.org/repos/asf/sentry/blob/908072d6/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONCreateDatabaseMessage.java ---------------------------------------------------------------------- diff --git a/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONCreateDatabaseMessage.java b/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONCreateDatabaseMessage.java new file mode 100644 index 0000000..ba19cbe --- /dev/null +++ b/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONCreateDatabaseMessage.java @@ -0,0 +1,44 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.sentry.binding.metastore.messaging.json; + +import org.apache.hive.hcatalog.messaging.json.JSONCreateDatabaseMessage; +import org.codehaus.jackson.annotate.JsonProperty; + +public class SentryJSONCreateDatabaseMessage extends JSONCreateDatabaseMessage { + @JsonProperty + String location; + + public SentryJSONCreateDatabaseMessage() { + } + + public SentryJSONCreateDatabaseMessage(String server, String servicePrincipal, String db, Long timestamp, String location) { + super(server, servicePrincipal, db, timestamp); + this.location = location; + } + + public String getLocation() { + return location; + } + + @Override + public String toString() { + return SentryJSONMessageDeserializer.serialize(this); + } +} http://git-wip-us.apache.org/repos/asf/sentry/blob/908072d6/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONCreateTableMessage.java ---------------------------------------------------------------------- diff --git a/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONCreateTableMessage.java b/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONCreateTableMessage.java new file mode 100644 index 0000000..57d11d2 --- /dev/null +++ b/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONCreateTableMessage.java @@ -0,0 +1,45 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.sentry.binding.metastore.messaging.json; + +import org.apache.hive.hcatalog.messaging.json.JSONCreateTableMessage; +import org.codehaus.jackson.annotate.JsonProperty; + +public class SentryJSONCreateTableMessage extends JSONCreateTableMessage { + @JsonProperty + String location; + + public SentryJSONCreateTableMessage() { + } + + public SentryJSONCreateTableMessage(String server, String servicePrincipal, String db, String table, Long timestamp, String location) { + super(server, servicePrincipal, db, table, timestamp); + this.location = location; + } + + public String getLocation() { + return location; + } + + @Override + public String toString() { + return SentryJSONMessageDeserializer.serialize(this); + } + +} http://git-wip-us.apache.org/repos/asf/sentry/blob/908072d6/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONDropDatabaseMessage.java ---------------------------------------------------------------------- diff --git a/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONDropDatabaseMessage.java b/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONDropDatabaseMessage.java new file mode 100644 index 0000000..05f83f7 --- /dev/null +++ b/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONDropDatabaseMessage.java @@ -0,0 +1,44 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.sentry.binding.metastore.messaging.json; + +import org.apache.hive.hcatalog.messaging.json.JSONDropDatabaseMessage; +import org.codehaus.jackson.annotate.JsonProperty; + +public class SentryJSONDropDatabaseMessage extends JSONDropDatabaseMessage{ + @JsonProperty + String location; + + public SentryJSONDropDatabaseMessage() { + } + + public SentryJSONDropDatabaseMessage(String server, String servicePrincipal, String db, Long timestamp, String location) { + super(server, servicePrincipal, db, timestamp); + this.location = location; + } + + public String getLocation() { + return location; + } + + @Override + public String toString() { + return SentryJSONMessageDeserializer.serialize(this); + } +} http://git-wip-us.apache.org/repos/asf/sentry/blob/908072d6/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONDropPartitionMessage.java ---------------------------------------------------------------------- diff --git a/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONDropPartitionMessage.java b/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONDropPartitionMessage.java new file mode 100644 index 0000000..2ab61f7 --- /dev/null +++ b/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONDropPartitionMessage.java @@ -0,0 +1,49 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.sentry.binding.metastore.messaging.json; + +import org.apache.hive.hcatalog.messaging.json.JSONDropPartitionMessage; +import org.codehaus.jackson.annotate.JsonProperty; + +import java.util.List; +import java.util.Map; + +public class SentryJSONDropPartitionMessage extends JSONDropPartitionMessage { + @JsonProperty + String location; + + public SentryJSONDropPartitionMessage() { + } + + public SentryJSONDropPartitionMessage(String server, String servicePrincipal, String db, String table, + List<Map<String, String>> partitions, Long timestamp, String location) { + super(server, servicePrincipal, db, table, partitions, timestamp); + this.location = location; + } + + public String getLocation() { + return location; + } + + @Override + public String toString() { + return SentryJSONMessageDeserializer.serialize(this); + } + +} http://git-wip-us.apache.org/repos/asf/sentry/blob/908072d6/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONDropTableMessage.java ---------------------------------------------------------------------- diff --git a/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONDropTableMessage.java b/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONDropTableMessage.java new file mode 100644 index 0000000..7005776 --- /dev/null +++ b/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONDropTableMessage.java @@ -0,0 +1,45 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.sentry.binding.metastore.messaging.json; + +import org.apache.hive.hcatalog.messaging.json.JSONDropTableMessage; +import org.codehaus.jackson.annotate.JsonProperty; + + +public class SentryJSONDropTableMessage extends JSONDropTableMessage { + @JsonProperty + String location; + + public SentryJSONDropTableMessage() { + } + + public SentryJSONDropTableMessage(String server, String servicePrincipal, String db, String table, Long timestamp, String location) { + super(server, servicePrincipal, db, table, timestamp); + this.location = location; + } + + public String getLocation() { + return location; + } + + @Override + public String toString() { + return SentryJSONMessageDeserializer.serialize(this); + } +} http://git-wip-us.apache.org/repos/asf/sentry/blob/908072d6/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONMessageDeserializer.java ---------------------------------------------------------------------- diff --git a/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONMessageDeserializer.java b/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONMessageDeserializer.java new file mode 100644 index 0000000..b645c45 --- /dev/null +++ b/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONMessageDeserializer.java @@ -0,0 +1,110 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.sentry.binding.metastore.messaging.json; + +import org.apache.hive.hcatalog.messaging.*; +import org.codehaus.jackson.map.DeserializationConfig; +import org.codehaus.jackson.map.ObjectMapper; + +public class SentryJSONMessageDeserializer extends MessageDeserializer { + static ObjectMapper mapper = new ObjectMapper(); + + public SentryJSONMessageDeserializer() { + } + + /** + * Method to de-serialize CreateDatabaseMessage instance. + */ + public SentryJSONCreateDatabaseMessage getCreateDatabaseMessage(String messageBody) { + try { + return (SentryJSONCreateDatabaseMessage)mapper.readValue(messageBody, SentryJSONCreateDatabaseMessage.class); + } catch (Exception var3) { + throw new IllegalArgumentException("Could not construct SentryJSONCreateDatabaseMessage.", var3); + } + } + + public SentryJSONDropDatabaseMessage getDropDatabaseMessage(String messageBody) { + try { + return (SentryJSONDropDatabaseMessage)mapper.readValue(messageBody, SentryJSONDropDatabaseMessage.class); + } catch (Exception var3) { + throw new IllegalArgumentException("Could not construct SentryJSONDropDatabaseMessage.", var3); + } + } + + public SentryJSONCreateTableMessage getCreateTableMessage(String messageBody) { + try { + return (SentryJSONCreateTableMessage)mapper.readValue(messageBody, SentryJSONCreateTableMessage.class); + } catch (Exception var3) { + throw new IllegalArgumentException("Could not construct SentryJSONCreateTableMessage.", var3); + } + } + + public SentryJSONAlterTableMessage getAlterTableMessage(String messageBody) { + try { + return (SentryJSONAlterTableMessage)mapper.readValue(messageBody, SentryJSONAlterTableMessage.class); + } catch (Exception var3) { + throw new IllegalArgumentException("Could not construct SentryJSONAlterTableMessage.", var3); + } + } + + public SentryJSONDropTableMessage getDropTableMessage(String messageBody) { + try { + return (SentryJSONDropTableMessage)mapper.readValue(messageBody, SentryJSONDropTableMessage.class); + } catch (Exception var3) { + throw new IllegalArgumentException("Could not construct SentryJSONDropTableMessage.", var3); + } + } + + public SentryJSONAddPartitionMessage getAddPartitionMessage(String messageBody) { + try { + return (SentryJSONAddPartitionMessage)mapper.readValue(messageBody, SentryJSONAddPartitionMessage.class); + } catch (Exception var3) { + throw new IllegalArgumentException("Could not construct SentryJSONAddPartitionMessage.", var3); + } + } + + public SentryJSONAlterPartitionMessage getAlterPartitionMessage(String messageBody) { + try { + return (SentryJSONAlterPartitionMessage)mapper.readValue(messageBody, SentryJSONAlterPartitionMessage.class); + } catch (Exception var3) { + throw new IllegalArgumentException("Could not construct SentryJSONAlterPartitionMessage.", var3); + } + } + + public SentryJSONDropPartitionMessage getDropPartitionMessage(String messageBody) { + try { + return (SentryJSONDropPartitionMessage)mapper.readValue(messageBody, SentryJSONDropPartitionMessage.class); + } catch (Exception var3) { + throw new IllegalArgumentException("Could not construct SentryJSONDropPartitionMessage.", var3); + } + } + + static { + mapper.configure(DeserializationConfig.Feature.FAIL_ON_UNKNOWN_PROPERTIES, false); + } + + public static String serialize(Object object) { + try { + return mapper.writeValueAsString(object); + } + catch (Exception exception) { + throw new IllegalArgumentException("Could not serialize: ", exception); + } + } +} http://git-wip-us.apache.org/repos/asf/sentry/blob/908072d6/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONMessageFactory.java ---------------------------------------------------------------------- diff --git a/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONMessageFactory.java b/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONMessageFactory.java new file mode 100644 index 0000000..00e7db8 --- /dev/null +++ b/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONMessageFactory.java @@ -0,0 +1,177 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sentry.binding.metastore.messaging.json; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hive.common.classification.InterfaceAudience; +import org.apache.hadoop.hive.common.classification.InterfaceStability; +import org.apache.hadoop.hive.metastore.api.Database; +import org.apache.hadoop.hive.metastore.api.FieldSchema; +import org.apache.hadoop.hive.metastore.api.Partition; +import org.apache.hadoop.hive.metastore.api.Table; +import org.apache.hadoop.hive.metastore.partition.spec.PartitionSpecProxy; +import org.apache.hive.hcatalog.messaging.*; + +import java.util.*; + +public class SentryJSONMessageFactory extends MessageFactory { + private static final Log LOG = LogFactory.getLog(SentryJSONMessageFactory.class.getName()); + private static SentryJSONMessageDeserializer deserializer = new SentryJSONMessageDeserializer(); + public SentryJSONMessageFactory() { + LOG.info("Using SentryJSONMessageFactory for building Notification log messages "); + + } + public MessageDeserializer getDeserializer() { + return deserializer; + } + + public String getVersion() { + return "0.1"; + } + + public String getMessageFormat() { + return "json"; + } + + public SentryJSONCreateDatabaseMessage buildCreateDatabaseMessage(Database db) { + return new SentryJSONCreateDatabaseMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, db.getName(), + Long.valueOf(this.now()), db.getLocationUri()); + } + public SentryJSONDropDatabaseMessage buildDropDatabaseMessage(Database db) { + return new SentryJSONDropDatabaseMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, db.getName(), + Long.valueOf(this.now()), db.getLocationUri()); + } + + public SentryJSONCreateTableMessage buildCreateTableMessage(Table table) { + return new SentryJSONCreateTableMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, table.getDbName(), + table.getTableName(), Long.valueOf(this.now()), table.getSd().getLocation()); + } + + public SentryJSONAlterTableMessage buildAlterTableMessage(Table before, Table after) { + return new SentryJSONAlterTableMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, before.getDbName(), + before.getTableName(), Long.valueOf(this.now()), before.getSd().getLocation(), after.getSd().getLocation()); + } + + public SentryJSONDropTableMessage buildDropTableMessage(Table table) { + return new SentryJSONDropTableMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, table.getDbName(), + table.getTableName(), Long.valueOf(this.now()), table.getSd().getLocation()); + } + + public SentryJSONAddPartitionMessage buildAddPartitionMessage(Table table, List<Partition> partitions) { + return new SentryJSONAddPartitionMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, table.getDbName(), + table.getTableName(), getPartitionKeyValues(table, partitions), Long.valueOf(this.now()), + getPartitionLocations(partitions)); + } + + private List<String> getPartitionLocations(List<Partition> partitions) { + List<String> paths = new ArrayList<String>(); + for(Partition partition:partitions) { + paths.add(partition.getSd().getLocation()); + } + return paths; + } + + //TODO: Not sure what is this used for. Need to investigate + private List<String> getPartitionLocations(PartitionSpecProxy partitionSpec) { + Iterator<Partition> iterator = partitionSpec.getPartitionIterator(); + List<String> locations = new ArrayList<String>(); + while(iterator.hasNext()) { + locations.add(iterator.next().getSd().getLocation()); + } + return locations; + } + + @InterfaceAudience.LimitedPrivate({"Hive"}) + @InterfaceStability.Evolving + public SentryJSONAddPartitionMessage buildAddPartitionMessage(Table table, PartitionSpecProxy partitionSpec) { + return new SentryJSONAddPartitionMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, table.getDbName(), + table.getTableName(), getPartitionKeyValues(table, partitionSpec), Long.valueOf(this.now()), + getPartitionLocations(partitionSpec)); + } + + public SentryJSONAlterPartitionMessage buildAlterPartitionMessage(Partition before, Partition after) { + /* + f (partitionEvent.getOldPartition() != null) { + oldLoc = partitionEvent.getOldPartition().getSd().getLocation(); + } + if (partitionEvent.getNewPartition() != null) { + newLoc = partitionEvent.getNewPartition().getSd().getLocation(); + } + + if ((oldLoc != null) && (newLoc != null) && (!oldLoc.equals(newLoc))) { + String authzObj = + partitionEvent.getOldPartition().getDbName() + "." + + partitionEvent.getOldPartition().getTableName(); + for (SentryMetastoreListenerPlugin plugin : sentryPlugins) { + plugin.renameAuthzObject(authzObj, oldLoc, + authzObj, newLoc); + } + } + * */ + return new SentryJSONAlterPartitionMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, before.getDbName(), + before.getTableName(), before.getValues(), Long.valueOf(this.now()), before.getSd().getLocation(), + after.getSd().getLocation()); + } + + public SentryJSONDropPartitionMessage buildDropPartitionMessage(Table table, Partition partition) { + return new SentryJSONDropPartitionMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, partition.getDbName(), + partition.getTableName(), Arrays.asList(getPartitionKeyValues(table, partition)), + Long.valueOf(this.now()), partition.getSd().getLocation()); + } + + private static Map<String, String> getPartitionKeyValues(Table table, Partition partition) { + LinkedHashMap partitionKeys = new LinkedHashMap(); + + for(int i = 0; i < table.getPartitionKeysSize(); ++i) { + partitionKeys.put(((FieldSchema)table.getPartitionKeys().get(i)).getName(), partition.getValues().get(i)); + } + + return partitionKeys; + } + + private static List<Map<String, String>> getPartitionKeyValues(Table table, List<Partition> partitions) { + ArrayList partitionList = new ArrayList(partitions.size()); + Iterator i$ = partitions.iterator(); + + while(i$.hasNext()) { + Partition partition = (Partition)i$.next(); + partitionList.add(getPartitionKeyValues(table, partition)); + } + + return partitionList; + } + + @InterfaceAudience.LimitedPrivate({"Hive"}) + @InterfaceStability.Evolving + private static List<Map<String, String>> getPartitionKeyValues(Table table, PartitionSpecProxy partitionSpec) { + ArrayList partitionList = new ArrayList(); + PartitionSpecProxy.PartitionIterator iterator = partitionSpec.getPartitionIterator(); + + while(iterator.hasNext()) { + Partition partition = (Partition)iterator.next(); + partitionList.add(getPartitionKeyValues(table, partition)); + } + + return partitionList; + } + //This is private in parent class + private long now() { + return System.currentTimeMillis() / 1000L; + } +} http://git-wip-us.apache.org/repos/asf/sentry/blob/908072d6/sentry-core/sentry-core-common/src/main/java/org/apache/sentry/core/common/exception/SentryInvalidHMSEventException.java ---------------------------------------------------------------------- diff --git a/sentry-core/sentry-core-common/src/main/java/org/apache/sentry/core/common/exception/SentryInvalidHMSEventException.java b/sentry-core/sentry-core-common/src/main/java/org/apache/sentry/core/common/exception/SentryInvalidHMSEventException.java new file mode 100644 index 0000000..50fb59c --- /dev/null +++ b/sentry-core/sentry-core-common/src/main/java/org/apache/sentry/core/common/exception/SentryInvalidHMSEventException.java @@ -0,0 +1,31 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sentry.core.common.exception; + +public class SentryInvalidHMSEventException extends SentryUserException { + private static final long serialVersionUID = 29620806553835L; + public SentryInvalidHMSEventException(String msg) { + super(msg); + } + public SentryInvalidHMSEventException(String msg, String reason) { + super(msg, reason); + } + public SentryInvalidHMSEventException(String msg, Throwable cause) { + super(msg, cause); + } +} http://git-wip-us.apache.org/repos/asf/sentry/blob/908072d6/sentry-core/sentry-core-common/src/main/java/org/apache/sentry/core/common/exception/SentryStandbyException.java ---------------------------------------------------------------------- diff --git a/sentry-core/sentry-core-common/src/main/java/org/apache/sentry/core/common/exception/SentryStandbyException.java b/sentry-core/sentry-core-common/src/main/java/org/apache/sentry/core/common/exception/SentryStandbyException.java index b2df699..da6cfce 100644 --- a/sentry-core/sentry-core-common/src/main/java/org/apache/sentry/core/common/exception/SentryStandbyException.java +++ b/sentry-core/sentry-core-common/src/main/java/org/apache/sentry/core/common/exception/SentryStandbyException.java @@ -34,4 +34,8 @@ public class SentryStandbyException extends SentryUserException { public SentryStandbyException(String msg, String reason) { super(msg, reason); } + + public SentryStandbyException(String msg, Throwable t) { + super(msg, t); + } } http://git-wip-us.apache.org/repos/asf/sentry/blob/908072d6/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/thrift/SentryPolicyServiceClientDefaultImpl.java ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/thrift/SentryPolicyServiceClientDefaultImpl.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/thrift/SentryPolicyServiceClientDefaultImpl.java index 1039e6e..4f42a51 100644 --- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/thrift/SentryPolicyServiceClientDefaultImpl.java +++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/thrift/SentryPolicyServiceClientDefaultImpl.java @@ -132,12 +132,18 @@ public class SentryPolicyServiceClientDefaultImpl implements SentryPolicyService } public SentryPolicyServiceClientDefaultImpl(Configuration conf) throws IOException { + this(Preconditions.checkNotNull(conf.get(ClientConfig.SERVER_RPC_ADDRESS), "Config key " + + ClientConfig.SERVER_RPC_ADDRESS + " is required"), conf.getInt( + ClientConfig.SERVER_RPC_PORT, ClientConfig.SERVER_RPC_PORT_DEFAULT), conf); + } + + public SentryPolicyServiceClientDefaultImpl(String addr, int port, + Configuration conf) throws IOException { this.conf = conf; Preconditions.checkNotNull(this.conf, "Configuration object cannot be null"); this.serverAddress = NetUtils.createSocketAddr(Preconditions.checkNotNull( - conf.get(ClientConfig.SERVER_RPC_ADDRESS), "Config key " - + ClientConfig.SERVER_RPC_ADDRESS + " is required"), conf.getInt( - ClientConfig.SERVER_RPC_PORT, ClientConfig.SERVER_RPC_PORT_DEFAULT)); + addr, "Config key " + ClientConfig.SERVER_RPC_ADDRESS + + " is required"), port); this.connectionTimeout = conf.getInt(ClientConfig.SERVER_RPC_CONN_TIMEOUT, ClientConfig.SERVER_RPC_CONN_TIMEOUT_DEFAULT); kerberos = ServerConfig.SECURITY_MODE_KERBEROS.equalsIgnoreCase( http://git-wip-us.apache.org/repos/asf/sentry/blob/908072d6/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/HMSFollower.java ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/HMSFollower.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/HMSFollower.java new file mode 100644 index 0000000..4430471 --- /dev/null +++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/HMSFollower.java @@ -0,0 +1,411 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sentry.service.thrift; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.metastore.HiveMetaStoreClient; +import org.apache.hadoop.hive.metastore.api.MetaException; +import org.apache.hadoop.hive.metastore.api.NotificationEvent; +import org.apache.hadoop.hive.metastore.api.NotificationEventResponse; +import org.apache.hadoop.security.SaslRpcServer; +import org.apache.hive.hcatalog.messaging.HCatEventMessage; +import org.apache.sentry.binding.hive.conf.HiveAuthzConf; +import org.apache.sentry.core.common.exception.*; +import org.apache.sentry.provider.db.service.persistent.SentryStore; +import org.apache.sentry.provider.db.service.thrift.TSentryAuthorizable; +import org.apache.thrift.TException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.apache.sentry.binding.metastore.messaging.json.*; + +import javax.security.auth.Subject; +import javax.security.auth.login.LoginException; +import java.io.File; +import java.io.IOException; +import java.security.PrivilegedActionException; +import java.security.PrivilegedExceptionAction; +import java.util.List; + +import static org.apache.sentry.binding.hive.conf.HiveAuthzConf.AuthzConfVars.AUTHZ_SYNC_CREATE_WITH_POLICY_STORE; +import static org.apache.sentry.binding.hive.conf.HiveAuthzConf.AuthzConfVars.AUTHZ_SYNC_DROP_WITH_POLICY_STORE; + +/* +HMSFollower is the thread which follows the Hive MetaStore state changes from Sentry. +It gets the full update and notification logs from HMS and applies it to +update permissions stored in Sentry using SentryStore and also update the <obj,path> state +stored for HDFS- Sentry sync. + */ +@SuppressWarnings("PMD") +public class HMSFollower implements Runnable { + private static final Logger LOGGER = LoggerFactory.getLogger(HMSFollower.class); + + private long currentEventID; + private static boolean connectedToHMS = false; + private HiveMetaStoreClient client; + private SentryKerberosContext kerberosContext; + private Configuration authzConf; + private boolean kerberos; + private SentryStore sentryStore; + private String hiveInstance; + final static int maxRetriesForLogin = 3; + final static int maxRetriesForConnection = 3; + + HMSFollower(Configuration conf) throws SentryNoSuchObjectException, + SentryAccessDeniedException, SentrySiteConfigurationException, IOException { //TODO: Handle any possible exceptions or throw specific exceptions + LOGGER.info("HMSFollower is being initialized"); + authzConf = conf; + try { + sentryStore = new SentryStore(authzConf); + } catch (SentryStandbyException e) { + //TODO: Do not start HMSFollower if standby + } + //TODO: Initialize currentEventID from Sentry db + currentEventID = 0; + } + + @VisibleForTesting + HMSFollower(Configuration conf, SentryStore sentryStore, String hiveInstance) { + this.authzConf = conf; + this.sentryStore = sentryStore; + this.hiveInstance = hiveInstance; + } + + @VisibleForTesting + public static boolean isConnectedToHMS() { + return connectedToHMS; + } + + /* + Returns HMS Client if successful, returns null if HMS is not ready yet to take connections + Throws @LoginException if Kerberos context creation failed using Sentry's kerberos credentials + Throws @MetaException if there was a problem on creating an HMSClient + */ + private HiveMetaStoreClient getMetaStoreClient(Configuration conf) + throws LoginException, MetaException { + if(client != null) { + return client; + } + // Seems like HMS client creation although seems successful, + // it actually connects to an invalid HMS instance. + // So it seems like it is necessary to wait until we make sure metastore config is properly loaded. + boolean loadedHiveConf = HiveConf.isLoadMetastoreConfig(); + if(!loadedHiveConf) { + return null; + } + final HiveConf hiveConf = new HiveConf(); + hiveInstance = hiveConf.get(HiveAuthzConf.AuthzConfVars.AUTHZ_SERVER_NAME.getVar()); + + String principal, keytab; + + //TODO: Is this the right(standard) way to create a HMS client? HiveMetastoreClientFactoryImpl? + //TODO: Check if HMS is using kerberos instead of relying on Sentry conf + //TODO: Handle TGT renewals + kerberos = ServiceConstants.ServerConfig.SECURITY_MODE_KERBEROS.equalsIgnoreCase( + conf.get(ServiceConstants.ServerConfig.SECURITY_MODE, ServiceConstants.ServerConfig.SECURITY_MODE_KERBEROS).trim()); + if (kerberos) { + LOGGER.info("Making a kerberos connection to HMS"); + //TODO: Is this needed? Use Hadoop libraries to translate the _HOST placeholder with actual hostname + //Validate principal + principal = Preconditions.checkNotNull(ServiceConstants.ServerConfig.PRINCIPAL, + ServiceConstants.ServerConfig.PRINCIPAL + " is required"); + LOGGER.info("Using kerberos principal: " + principal); + final String[] principalParts = SaslRpcServer.splitKerberosName(principal); + Preconditions.checkArgument(principalParts.length == 3, + "Kerberos principal should have 3 parts: " + principal); + + keytab = Preconditions.checkNotNull(conf.get(ServiceConstants.ServerConfig.KEY_TAB), + ServiceConstants.ServerConfig.KEY_TAB + " is required"); + File keytabFile = new File(keytab); + Preconditions.checkState(keytabFile.isFile() && keytabFile.canRead(), + "Keytab " + keytab + " does not exist or is not readable."); + boolean establishedKerberosContext = false; + int attempt = 1; + while(establishedKerberosContext) { + try { + kerberosContext = new SentryKerberosContext(principal, keytab, true); + establishedKerberosContext = true; + LOGGER.info("Established kerberos context, will now connect to HMS"); + } catch (LoginException e) { + //Kerberos login failed + if( attempt > maxRetriesForLogin ) { + throw e; + } + attempt++; + } + } + boolean establishedConnection = false; + attempt = 1; + while(establishedConnection) { + try { + client = Subject.doAs(kerberosContext.getSubject(), new PrivilegedExceptionAction<HiveMetaStoreClient>() { + @Override + public HiveMetaStoreClient run() throws Exception { + return new HiveMetaStoreClient(hiveConf); + } + }); + LOGGER.info("Secure connection established with HMS"); + } catch (PrivilegedActionException e) { + if( attempt > maxRetriesForConnection ) { + //We should just retry as it is possible that HMS is not ready yet to receive requests + //TODO: How do we differentiate between kerberos problem versus HMS not being up? + LOGGER.error("Cannot connect to HMS", e); + } + attempt++; + } + } + } else { + //This is only for testing purposes. Sentry strongly recommends strong authentication + client = new HiveMetaStoreClient(hiveConf); + LOGGER.info("Non secure connection established with HMS"); + } + return client; + } + + public void run() { + if( client == null ) { + try { + client = getMetaStoreClient(authzConf); + if (client == null) { + //TODO: Do we want to throw an exception after a certain timeout? + return; + } else { + connectedToHMS = true; + LOGGER.info("HMSFollower of Sentry successfully connected to HMS"); + } + } catch (Exception e) { + + } + } + if (needFullUpdate()) { + //TODO: Handle + } else { + try { + NotificationEventResponse response = client.getNextNotification(currentEventID, Integer.MAX_VALUE, null); + if (response.isSetEvents()) { + LOGGER.info(String.format("CurrentEventID = %s. Processing %s events", + currentEventID, response.getEvents().size())); + processNotificationEvents(response.getEvents()); + } + } catch (TException e) { + LOGGER.error("ThriftException occured fetching Notification entries, will try"); + e.printStackTrace(); + } catch (SentryInvalidInputException|SentryInvalidHMSEventException e) { + throw new RuntimeException(e); + } + } + } + + private boolean needFullUpdate() { + //TODO Implement + return false; + } + + private boolean syncWithPolicyStore(HiveAuthzConf.AuthzConfVars syncConfVar) { + return "true" + .equalsIgnoreCase((authzConf.get(syncConfVar.getVar(), "true"))); + } + + /* + Throws SentryInvalidHMSEventException if Notification event contains insufficient information + */ + + void processNotificationEvents(List<NotificationEvent> events) throws + SentryInvalidHMSEventException, SentryInvalidInputException { + SentryJSONMessageDeserializer deserializer = new SentryJSONMessageDeserializer(); + + for (NotificationEvent event : events) { + String dbName, tableName, oldLocation, newLocation, location; + switch (HCatEventMessage.EventType.valueOf(event.getEventType())) { + case CREATE_DATABASE: + SentryJSONCreateDatabaseMessage message = deserializer.getCreateDatabaseMessage(event.getMessage()); + dbName = message.getDB(); + + location = message.getLocation(); + if (dbName == null || location == null) { + throw new SentryInvalidHMSEventException(String.format("Create database event has incomplete information. " + + "dbName = %s location = %s", dbName, location)); + } + if (syncWithPolicyStore(AUTHZ_SYNC_CREATE_WITH_POLICY_STORE)) { + try { + dropSentryDbPrivileges(dbName); + } catch (SentryNoSuchObjectException e) { + LOGGER.info("Drop Sentry privilege ignored as there are no privileges on the database: %s", dbName); + } catch (SentryInvalidInputException e) { + throw new SentryInvalidInputException("Could not process Create database event. Event: " + event.toString(), e); + } + } + //TODO: HDFSPlugin.addPath(dbName, location) + break; + case DROP_DATABASE: + SentryJSONDropDatabaseMessage dropDatabaseMessage = deserializer.getDropDatabaseMessage(event.getMessage()); + dbName = dropDatabaseMessage.getDB(); + if (dbName == null) { + throw new SentryInvalidHMSEventException(String.format("Drop database event has incomplete information. " + + "dbName = %s", dbName)); + } + if (syncWithPolicyStore(AUTHZ_SYNC_DROP_WITH_POLICY_STORE)) { + try { + dropSentryDbPrivileges(dbName); + } catch (SentryNoSuchObjectException e) { + LOGGER.info("Drop Sentry privilege ignored as there are no privileges on the database: %s", dbName); + } catch (SentryInvalidInputException e) { + throw new SentryInvalidInputException("Could not process Drop database event. Event: " + event.toString(), e); + } + } + //TODO: HDFSPlugin.deletePath(dbName, location) + break; + case CREATE_TABLE: + SentryJSONCreateTableMessage createTableMessage = deserializer.getCreateTableMessage(event.getMessage()); + dbName = createTableMessage.getDB(); + tableName = createTableMessage.getTable(); + location = createTableMessage.getLocation(); + if (dbName == null || tableName == null || location == null) { + throw new SentryInvalidHMSEventException(String.format("Create table event has incomplete information. " + + "dbName = %s, tableName = %s, location = %s", dbName, tableName, location)); + } + if (syncWithPolicyStore(AUTHZ_SYNC_CREATE_WITH_POLICY_STORE)) { + try { + dropSentryTablePrivileges(dbName, tableName); + } catch (SentryNoSuchObjectException e) { + LOGGER.info("Drop Sentry privilege ignored as there are no privileges on the table: %s.%s", dbName, tableName); + } catch (SentryInvalidInputException e) { + throw new SentryInvalidInputException("Could not process Create table event. Event: " + event.toString(), e); + } + } + //TODO: HDFSPlugin.deletePath(dbName, location) + break; + case DROP_TABLE: + SentryJSONDropTableMessage dropTableMessage = deserializer.getDropTableMessage(event.getMessage()); + dbName = dropTableMessage.getDB(); + tableName = dropTableMessage.getTable(); + if (dbName == null || tableName == null) { + throw new SentryInvalidHMSEventException(String.format("Drop table event has incomplete information. " + + "dbName = %s, tableName = %s", dbName, tableName)); + } + if (syncWithPolicyStore(AUTHZ_SYNC_DROP_WITH_POLICY_STORE)) { + try{ + dropSentryTablePrivileges(dbName, tableName); + } catch (SentryNoSuchObjectException e) { + LOGGER.info("Drop Sentry privilege ignored as there are no privileges on the table: %s.%s", dbName, tableName); + } catch (SentryInvalidInputException e) { + throw new SentryInvalidInputException("Could not process Drop table event. Event: " + event.toString(), e); + } + } + //TODO: HDFSPlugin.deletePath(dbName, location) + break; + case ALTER_TABLE: + SentryJSONAlterTableMessage alterTableMessage = deserializer.getAlterTableMessage(event.getMessage()); + + String oldDbName = alterTableMessage.getDB(); + String oldTableName = alterTableMessage.getTable(); + String newDbName = event.getDbName(); + String newTableName = event.getTableName(); + oldLocation = alterTableMessage.getOldLocation(); + newLocation = alterTableMessage.getLocation(); + + if (oldDbName == null || oldTableName == null || newDbName == null || newTableName == null || + oldLocation == null || newLocation == null) { + throw new SentryInvalidHMSEventException(String.format("Alter table event has incomplete information. " + + "oldDbName = %s, oldTableName = %s, oldLocation = %s, newDbName = %s, newTableName = %s, newLocation = %s", + oldDbName, oldTableName, oldLocation, newDbName, newTableName, newLocation)); + } + + if(!newDbName.equalsIgnoreCase(oldDbName) || !oldTableName.equalsIgnoreCase(newTableName)) { // Name has changed + if(!oldLocation.equals(newLocation)) { // Location has changed + + //Name and path has changed + // - Alter table rename for managed table + //TODO: Handle HDFS plugin + + } else { + //Only name has changed + // - Alter table rename for an external table + //TODO: Handle HDFS plugin + + } + try { + renamePrivileges(oldDbName, oldTableName, newDbName, newTableName); + } catch (SentryNoSuchObjectException e) { + LOGGER.info("Rename Sentry privilege ignored as there are no privileges on the table: %s.%s", oldDbName, oldTableName); + } catch (SentryInvalidInputException e) { + throw new SentryInvalidInputException("Could not process Alter table event. Event: " + event.toString(), e); + } catch (SentryStandbyException e) { + LOGGER.error("Seems like this process became a standby. " + + "Ignoring the failure. The new leader will reprocess this notification event"); + } + } else if(!oldLocation.equals(newLocation)) { // Only Location has changed{ + //- Alter table set location + //TODO: Handle HDFS plugin + } else { + LOGGER.info(String.format("Alter table notification ignored as neither name nor location has changed: " + + "oldDbName = %s, oldTableName = %s, oldLocation = %s, newDbName = %s, newTableName = %s, newLocation = %s", + oldDbName, oldTableName, oldLocation, newDbName, newTableName, newLocation)); + } + //TODO: Write test cases for all these cases + break; + case ADD_PARTITION: + case DROP_PARTITION: + case ALTER_PARTITION: + //TODO: Handle HDFS plugin + break; + } + currentEventID = event.getEventId(); + } + } + + private void dropSentryDbPrivileges(String dbName) throws SentryNoSuchObjectException, SentryInvalidInputException { + TSentryAuthorizable authorizable = new TSentryAuthorizable(hiveInstance); + authorizable.setDb(dbName); + try { + sentryStore.dropPrivilege(authorizable); + } catch (SentryStandbyException e) { + LOGGER.error("Seems like this process became a standby. " + + "Ignoring the failure. The new leader will reprocess this notification event"); + } + } + private void dropSentryTablePrivileges(String dbName, String tableName) throws SentryNoSuchObjectException, + SentryInvalidInputException { + TSentryAuthorizable authorizable = new TSentryAuthorizable(hiveInstance); + authorizable.setDb(dbName); + authorizable.setTable(tableName); + try { + sentryStore.dropPrivilege(authorizable); + } catch (SentryStandbyException e) { + LOGGER.error("Seems like this process became a standby. " + + "Ignoring the failure. The new leader will reprocess this notification event"); + } + } + private void renamePrivileges(String oldDbName, String oldTableName, String newDbName, String newTableName) throws + SentryNoSuchObjectException, SentryInvalidInputException, SentryStandbyException{ + TSentryAuthorizable oldAuthorizable = new TSentryAuthorizable(hiveInstance); + oldAuthorizable.setDb(oldDbName); + oldAuthorizable.setTable(oldTableName); + TSentryAuthorizable newAuthorizable = new TSentryAuthorizable(hiveInstance); + newAuthorizable.setDb(newDbName); + newAuthorizable.setTable(newTableName); + try { + sentryStore.renamePrivilege(oldAuthorizable, newAuthorizable); + } catch (SentryStandbyException e) { + LOGGER.error("Seems like this process became a standby. " + + "Ignoring the failure. The new leader will reprocess this notification event"); + } + } +}
