SENTRY-1321: Implement HMSFollower in Sentry service which reads the NotificationLog entries (Sravya Tirukkovalur, Reviewed by: Hao Hao)
Change-Id: Ibf06dcad6b68d900c5471ed092f2e3f3c820b8bb Project: http://git-wip-us.apache.org/repos/asf/sentry/repo Commit: http://git-wip-us.apache.org/repos/asf/sentry/commit/5a96dcda Tree: http://git-wip-us.apache.org/repos/asf/sentry/tree/5a96dcda Diff: http://git-wip-us.apache.org/repos/asf/sentry/diff/5a96dcda Branch: refs/heads/sentry-ha-redesign Commit: 5a96dcdae86f99ed5218e1443daf47411cec536e Parents: 39c473c Author: Sravya Tirukkovalur <[email protected]> Authored: Thu Aug 18 16:36:37 2016 -0700 Committer: Sravya Tirukkovalur <[email protected]> Committed: Thu Aug 18 16:36:37 2016 -0700 ---------------------------------------------------------------------- sentry-binding/pom.xml | 3 + .../sentry-binding-hive-common/pom.xml | 5 + .../sentry/binding/hive/conf/HiveAuthzConf.java | 269 ------------ sentry-binding/sentry-binding-hive-conf/pom.xml | 62 +++ .../sentry/binding/hive/conf/HiveAuthzConf.java | 269 ++++++++++++ .../sentry-binding-hive-follower/pom.xml | 71 ++++ .../json/JSONAlterPartitionMessage.java | 78 ++++ .../messaging/json/JSONAlterTableMessage.java | 68 +++ .../json/SentryJSONAddPartitionMessage.java | 49 +++ .../json/SentryJSONAlterPartitionMessage.java | 53 +++ .../json/SentryJSONAlterTableMessage.java | 50 +++ .../json/SentryJSONCreateDatabaseMessage.java | 44 ++ .../json/SentryJSONCreateTableMessage.java | 45 ++ .../json/SentryJSONDropDatabaseMessage.java | 44 ++ .../json/SentryJSONDropPartitionMessage.java | 49 +++ .../json/SentryJSONDropTableMessage.java | 45 ++ .../json/SentryJSONMessageDeserializer.java | 110 +++++ .../json/SentryJSONMessageFactory.java | 177 ++++++++ .../json/JSONAlterPartitionMessage.java | 78 ---- .../messaging/json/JSONAlterTableMessage.java | 68 --- .../json/SentryJSONAddPartitionMessage.java | 49 --- .../json/SentryJSONAlterPartitionMessage.java | 53 --- .../json/SentryJSONAlterTableMessage.java | 50 --- .../json/SentryJSONCreateDatabaseMessage.java | 44 -- .../json/SentryJSONCreateTableMessage.java | 45 -- .../json/SentryJSONDropDatabaseMessage.java | 44 -- .../json/SentryJSONDropPartitionMessage.java | 49 --- .../json/SentryJSONDropTableMessage.java | 45 -- .../json/SentryJSONMessageDeserializer.java | 110 ----- .../json/SentryJSONMessageFactory.java | 177 -------- .../SentryInvalidHMSEventException.java | 31 ++ .../exception/SentryInvalidInputException.java | 3 + sentry-provider/sentry-provider-db/pom.xml | 31 ++ .../sentry/service/thrift/HMSFollower.java | 411 +++++++++++++++++++ .../service/thrift/SentryKerberosContext.java | 10 +- .../sentry/service/thrift/SentryService.java | 22 +- .../sentry/service/thrift/TestHMSFollower.java | 165 ++++++++ .../TestDbPrivilegeCleanupOnDrop.java | 25 +- .../AbstractTestWithStaticConfiguration.java | 7 +- .../e2e/hive/hiveserver/HiveServerFactory.java | 4 + 40 files changed, 1912 insertions(+), 1100 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sentry/blob/5a96dcda/sentry-binding/pom.xml ---------------------------------------------------------------------- diff --git a/sentry-binding/pom.xml b/sentry-binding/pom.xml index 30bca14..48ae245 100644 --- a/sentry-binding/pom.xml +++ b/sentry-binding/pom.xml @@ -34,6 +34,9 @@ limitations under the License. <module>sentry-binding-hive-common</module> <module>sentry-binding-solr</module> <module>sentry-binding-sqoop</module> + <module>sentry-binding-hive-follower</module> + <module>sentry-binding-hive-conf</module> + </modules> <profiles> http://git-wip-us.apache.org/repos/asf/sentry/blob/5a96dcda/sentry-binding/sentry-binding-hive-common/pom.xml ---------------------------------------------------------------------- diff --git a/sentry-binding/sentry-binding-hive-common/pom.xml b/sentry-binding/sentry-binding-hive-common/pom.xml index 18b422d..d38a494 100644 --- a/sentry-binding/sentry-binding-hive-common/pom.xml +++ b/sentry-binding/sentry-binding-hive-common/pom.xml @@ -78,6 +78,11 @@ limitations under the License. <scope>provided</scope> </dependency> <dependency> + <groupId>org.apache.sentry</groupId> + <artifactId>sentry-binding-hive-conf</artifactId> + <version>1.8.0-SNAPSHOT</version> + </dependency> + <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-client</artifactId> <version>${hadoop.version}</version> http://git-wip-us.apache.org/repos/asf/sentry/blob/5a96dcda/sentry-binding/sentry-binding-hive-common/src/main/java/org/apache/sentry/binding/hive/conf/HiveAuthzConf.java ---------------------------------------------------------------------- diff --git a/sentry-binding/sentry-binding-hive-common/src/main/java/org/apache/sentry/binding/hive/conf/HiveAuthzConf.java b/sentry-binding/sentry-binding-hive-common/src/main/java/org/apache/sentry/binding/hive/conf/HiveAuthzConf.java deleted file mode 100644 index ad19b37..0000000 --- a/sentry-binding/sentry-binding-hive-common/src/main/java/org/apache/sentry/binding/hive/conf/HiveAuthzConf.java +++ /dev/null @@ -1,269 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.sentry.binding.hive.conf; - -import java.net.MalformedURLException; -import java.net.URL; -import java.util.HashMap; -import java.util.Map; -import java.util.Map.Entry; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hive.conf.HiveConf; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - - -public class HiveAuthzConf extends Configuration { - - /** - * Configuration key used in hive-site.xml to point at sentry-site.xml - */ - public static final String HIVE_ACCESS_CONF_URL = "hive.access.conf.url"; - public static final String HIVE_SENTRY_CONF_URL = "hive.sentry.conf.url"; - public static final String HIVE_ACCESS_SUBJECT_NAME = "hive.access.subject.name"; - public static final String HIVE_SENTRY_SUBJECT_NAME = "hive.sentry.subject.name"; - public static final String HIVE_SENTRY_AUTH_ERRORS = "sentry.hive.authorization.errors"; - public static final String HIVE_SENTRY_MOCK_COMPILATION = "sentry.hive.mock.compilation"; - public static final String HIVE_SENTRY_MOCK_ERROR = "sentry.hive.mock.error"; - public static final String HIVE_SENTRY_PRIVILEGE_ERROR_MESSAGE = "No valid privileges"; - /** - * Property used to persist the role set in the session. This is not public for now. - */ - public static final String SENTRY_ACTIVE_ROLE_SET = "hive.sentry.active.role.set"; - - public static final String HIVE_SENTRY_SECURITY_COMMAND_WHITELIST = - "hive.sentry.security.command.whitelist"; - public static final String HIVE_SENTRY_SECURITY_COMMAND_WHITELIST_DEFAULT = - "set,reset,reload"; - - public static final String HIVE_SENTRY_SERDE_WHITELIST = "hive.sentry.serde.whitelist"; - public static final String HIVE_SENTRY_SERDE_WHITELIST_DEFAULT = "org.apache.hadoop.hive.serde2"; - - // Disable the serde Uri privileges by default for backward compatibilities. - public static final String HIVE_SENTRY_SERDE_URI_PRIVILIEGES_ENABLED = "hive.sentry.turn.on.serde.uri.privileges"; - public static final boolean HIVE_SENTRY_SERDE_URI_PRIVILIEGES_ENABLED_DEFAULT = false; - - public static final String HIVE_UDF_WHITE_LIST = - "concat,substr,substring,space,repeat,ascii,lpad,rpad,size,round,floor,sqrt,ceil," + - "ceiling,rand,abs,pmod,ln,log2,sin,asin,cos,acos,log10,log,exp,power,pow,sign,pi," + - "degrees,radians,atan,tan,e,conv,bin,hex,unhex,base64,unbase64,encode,decode,upper," + - "lower,ucase,lcase,trim,ltrim,rtrim,length,reverse,field,find_in_set,initcap,like," + - "rlike,regexp,regexp_replace,regexp_extract,parse_url,nvl,split,str_to_map,translate" + - ",positive,negative,day,dayofmonth,month,year,hour,minute,second,from_unixtime," + - "to_date,weekofyear,last_day,date_add,date_sub,datediff,add_months,get_json_object," + - "xpath_string,xpath_boolean,xpath_number,xpath_double,xpath_float,xpath_long," + - "xpath_int,xpath_short,xpath,+,-,*,/,%,div,&,|,^,~,current_database,isnull," + - "isnotnull,if,in,and,or,=,==,<=>,!=,<>,<,<=,>,>=,not,!,between,ewah_bitmap_and," + - "ewah_bitmap_or,ewah_bitmap_empty,boolean,tinyint,smallint,int,bigint,float,double," + - "string,date,timestamp,binary,decimal,varchar,char,max,min,sum,count,avg,std,stddev," + - "stddev_pop,stddev_samp,variance,var_pop,var_samp,covar_pop,covar_samp,corr," + - "histogram_numeric,percentile_approx,collect_set,collect_list,ngrams," + - "context_ngrams,ewah_bitmap,compute_stats,percentile," + - "array,assert_true,map,struct,named_struct,create_union,case,when,hash,coalesce," + - "index,in_file,instr,locate,elt,concat_ws,sort_array," + - "array_contains,sentences,map_keys,map_values,format_number,printf,greatest,least," + - "from_utc_timestamp,to_utc_timestamp,unix_timestamp,to_unix_timestamp,explode," + - "inline,json_tuple,parse_url_tuple,posexplode,stack,lead,lag,row_number,rank," + - "dense_rank,percent_rank,cume_dist,ntile,first_value,last_value,noop,noopwithmap," + - "noopstreaming,noopwithmapstreaming,windowingtablefunction,matchpath"; - - public static final String HIVE_UDF_BLACK_LIST = "reflect,reflect2,java_method"; - - /** - * Config setting definitions - */ - public static enum AuthzConfVars { - AUTHZ_PROVIDER("sentry.provider", - "org.apache.sentry.provider.common.HadoopGroupResourceAuthorizationProvider"), - AUTHZ_PROVIDER_RESOURCE("sentry.hive.provider.resource", ""), - AUTHZ_PROVIDER_BACKEND("sentry.hive.provider.backend", "org.apache.sentry.provider.file.SimpleFileProviderBackend"), - AUTHZ_POLICY_ENGINE("sentry.hive.policy.engine", "org.apache.sentry.policy.engine.common.CommonPolicyEngine"), - AUTHZ_POLICY_FILE_FORMATTER( - "sentry.hive.policy.file.formatter", - "org.apache.sentry.binding.hive.SentryIniPolicyFileFormatter"), - AUTHZ_SERVER_NAME("sentry.hive.server", ""), - AUTHZ_RESTRICT_DEFAULT_DB("sentry.hive.restrict.defaultDB", "false"), - SENTRY_TESTING_MODE("sentry.hive.testing.mode", "false"), - AUTHZ_ALLOW_HIVE_IMPERSONATION("sentry.hive.allow.hive.impersonation", "false"), - AUTHZ_ONFAILURE_HOOKS("sentry.hive.failure.hooks", ""), - AUTHZ_METASTORE_SERVICE_USERS("sentry.metastore.service.users", null), - AUTHZ_SYNC_ALTER_WITH_POLICY_STORE("sentry.hive.sync.alter", "true"), - AUTHZ_SYNC_CREATE_WITH_POLICY_STORE("sentry.hive.sync.create", "false"), - AUTHZ_SYNC_DROP_WITH_POLICY_STORE("sentry.hive.sync.drop", "true"), - - AUTHZ_PROVIDER_DEPRECATED("hive.sentry.provider", - "org.apache.sentry.provider.file.ResourceAuthorizationProvider"), - AUTHZ_PROVIDER_RESOURCE_DEPRECATED("hive.sentry.provider.resource", ""), - AUTHZ_SERVER_NAME_DEPRECATED("hive.sentry.server", ""), - AUTHZ_RESTRICT_DEFAULT_DB_DEPRECATED("hive.sentry.restrict.defaultDB", "false"), - SENTRY_TESTING_MODE_DEPRECATED("hive.sentry.testing.mode", "false"), - AUTHZ_ALLOW_HIVE_IMPERSONATION_DEPRECATED("hive.sentry.allow.hive.impersonation", "false"), - AUTHZ_ONFAILURE_HOOKS_DEPRECATED("hive.sentry.failure.hooks", ""); - - private final String varName; - private final String defaultVal; - - AuthzConfVars(String varName, String defaultVal) { - this.varName = varName; - this.defaultVal = defaultVal; - } - - public String getVar() { - return varName; - } - - public String getDefault() { - return defaultVal; - } - - public static String getDefault(String varName) { - for (AuthzConfVars oneVar : AuthzConfVars.values()) { - if(oneVar.getVar().equalsIgnoreCase(varName)) { - return oneVar.getDefault(); - } - } - return null; - } - } - - // map of current property names - > deprecated property names. - // The binding layer code should work if the deprecated property names are provided, - // as long as the new property names aren't also provided. Since the binding code - // only calls the new property names, we require a map from current names to deprecated - // names in order to check if the deprecated name of a property was set. - private static final Map<String, AuthzConfVars> currentToDeprecatedProps = - new HashMap<String, AuthzConfVars>(); - static { - currentToDeprecatedProps.put(AuthzConfVars.AUTHZ_PROVIDER.getVar(), AuthzConfVars.AUTHZ_PROVIDER_DEPRECATED); - currentToDeprecatedProps.put(AuthzConfVars.AUTHZ_PROVIDER_RESOURCE.getVar(), AuthzConfVars.AUTHZ_PROVIDER_RESOURCE_DEPRECATED); - currentToDeprecatedProps.put(AuthzConfVars.AUTHZ_SERVER_NAME.getVar(), AuthzConfVars.AUTHZ_SERVER_NAME_DEPRECATED); - currentToDeprecatedProps.put(AuthzConfVars.AUTHZ_RESTRICT_DEFAULT_DB.getVar(), AuthzConfVars.AUTHZ_RESTRICT_DEFAULT_DB_DEPRECATED); - currentToDeprecatedProps.put(AuthzConfVars.SENTRY_TESTING_MODE.getVar(), AuthzConfVars.SENTRY_TESTING_MODE_DEPRECATED); - currentToDeprecatedProps.put(AuthzConfVars.AUTHZ_ALLOW_HIVE_IMPERSONATION.getVar(), AuthzConfVars.AUTHZ_ALLOW_HIVE_IMPERSONATION_DEPRECATED); - currentToDeprecatedProps.put(AuthzConfVars.AUTHZ_ONFAILURE_HOOKS.getVar(), AuthzConfVars.AUTHZ_ONFAILURE_HOOKS_DEPRECATED); - }; - - private static final Logger LOG = LoggerFactory - .getLogger(HiveAuthzConf.class); - public static final String AUTHZ_SITE_FILE = "sentry-site.xml"; - private final String hiveAuthzSiteFile; - - public HiveAuthzConf(URL hiveAuthzSiteURL) { - super(); - LOG.info("DefaultFS: " + super.get("fs.defaultFS")); - addResource(hiveAuthzSiteURL); - applySystemProperties(); - LOG.info("DefaultFS: " + super.get("fs.defaultFS")); - this.hiveAuthzSiteFile = hiveAuthzSiteURL.toString(); - } - /** - * Apply system properties to this object if the property name is defined in ConfVars - * and the value is non-null and not an empty string. - */ - private void applySystemProperties() { - Map<String, String> systemProperties = getConfSystemProperties(); - for (Entry<String, String> systemProperty : systemProperties.entrySet()) { - this.set(systemProperty.getKey(), systemProperty.getValue()); - } - } - - /** - * This method returns a mapping from config variable name to its value for all config variables - * which have been set using System properties - */ - public static Map<String, String> getConfSystemProperties() { - Map<String, String> systemProperties = new HashMap<String, String>(); - - for (AuthzConfVars oneVar : AuthzConfVars.values()) { - String value = System.getProperty(oneVar.getVar()); - if (value != null && value.length() > 0) { - systemProperties.put(oneVar.getVar(), value); - } - } - return systemProperties; - } - - @Override - public String get(String varName) { - return get(varName, null); - } - - @Override - public String get(String varName, String defaultVal) { - String retVal = super.get(varName); - if (retVal == null) { - // check if the deprecated value is set here - if (currentToDeprecatedProps.containsKey(varName)) { - retVal = super.get(currentToDeprecatedProps.get(varName).getVar()); - } - if (retVal == null) { - retVal = AuthzConfVars.getDefault(varName); - } else { - LOG.warn("Using the deprecated config setting " + currentToDeprecatedProps.get(varName).getVar() + - " instead of " + varName); - } - } - if (retVal == null) { - retVal = defaultVal; - } - return retVal; - } - - public String getHiveAuthzSiteFile() { - return hiveAuthzSiteFile; - } - - /** - * Extract the authz config file path from given hive conf and load the authz config - * @param hiveConf - * @return - * @throws IllegalArgumentException - */ - public static HiveAuthzConf getAuthzConf(HiveConf hiveConf) - throws IllegalArgumentException { - boolean depreicatedConfigFile = false; - - String hiveAuthzConf = hiveConf.get(HiveAuthzConf.HIVE_SENTRY_CONF_URL); - if (hiveAuthzConf == null - || (hiveAuthzConf = hiveAuthzConf.trim()).isEmpty()) { - hiveAuthzConf = hiveConf.get(HiveAuthzConf.HIVE_ACCESS_CONF_URL); - depreicatedConfigFile = true; - } - - if (hiveAuthzConf == null - || (hiveAuthzConf = hiveAuthzConf.trim()).isEmpty()) { - throw new IllegalArgumentException("Configuration key " - + HiveAuthzConf.HIVE_SENTRY_CONF_URL + " value '" + hiveAuthzConf - + "' is invalid."); - } - - try { - return new HiveAuthzConf(new URL(hiveAuthzConf)); - } catch (MalformedURLException e) { - if (depreicatedConfigFile) { - throw new IllegalArgumentException("Configuration key " - + HiveAuthzConf.HIVE_ACCESS_CONF_URL - + " specifies a malformed URL '" + hiveAuthzConf + "'", e); - } else { - throw new IllegalArgumentException("Configuration key " - + HiveAuthzConf.HIVE_SENTRY_CONF_URL - + " specifies a malformed URL '" + hiveAuthzConf + "'", e); - } - } - } -} http://git-wip-us.apache.org/repos/asf/sentry/blob/5a96dcda/sentry-binding/sentry-binding-hive-conf/pom.xml ---------------------------------------------------------------------- diff --git a/sentry-binding/sentry-binding-hive-conf/pom.xml b/sentry-binding/sentry-binding-hive-conf/pom.xml new file mode 100644 index 0000000..2cae541 --- /dev/null +++ b/sentry-binding/sentry-binding-hive-conf/pom.xml @@ -0,0 +1,62 @@ +<?xml version="1.0"?> +<!-- +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +--> +<project xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"; xmlns="http://maven.apache.org/POM/4.0.0"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";> + <modelVersion>4.0.0</modelVersion> + + <parent> + <groupId>org.apache.sentry</groupId> + <artifactId>sentry-binding</artifactId> + <version>1.8.0-SNAPSHOT</version> + </parent> + + <artifactId>sentry-binding-hive-conf</artifactId> + <name>Sentry Hive Binding Configuration</name> + + <dependencies> + <dependency> + <groupId>org.apache.thrift</groupId> + <artifactId>libthrift</artifactId> + </dependency> + <dependency> + <groupId>org.apache.derby</groupId> + <artifactId>derby</artifactId> + </dependency> + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.apache.hive</groupId> + <artifactId>hive-exec</artifactId> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.apache.hive</groupId> + <artifactId>hive-service</artifactId> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-common</artifactId> + <scope>provided</scope> + </dependency> + </dependencies> + +</project> http://git-wip-us.apache.org/repos/asf/sentry/blob/5a96dcda/sentry-binding/sentry-binding-hive-conf/src/main/java/org/apache/sentry/binding/hive/conf/HiveAuthzConf.java ---------------------------------------------------------------------- diff --git a/sentry-binding/sentry-binding-hive-conf/src/main/java/org/apache/sentry/binding/hive/conf/HiveAuthzConf.java b/sentry-binding/sentry-binding-hive-conf/src/main/java/org/apache/sentry/binding/hive/conf/HiveAuthzConf.java new file mode 100644 index 0000000..4de755f --- /dev/null +++ b/sentry-binding/sentry-binding-hive-conf/src/main/java/org/apache/sentry/binding/hive/conf/HiveAuthzConf.java @@ -0,0 +1,269 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sentry.binding.hive.conf; + +import java.net.MalformedURLException; +import java.net.URL; +import java.util.HashMap; +import java.util.Map; +import java.util.Map.Entry; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.conf.HiveConf; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + + +public class HiveAuthzConf extends Configuration { + + /** + * Configuration key used in hive-site.xml to point at sentry-site.xml + */ + public static final String HIVE_ACCESS_CONF_URL = "hive.access.conf.url"; + public static final String HIVE_SENTRY_CONF_URL = "hive.sentry.conf.url"; + public static final String HIVE_ACCESS_SUBJECT_NAME = "hive.access.subject.name"; + public static final String HIVE_SENTRY_SUBJECT_NAME = "hive.sentry.subject.name"; + public static final String HIVE_SENTRY_AUTH_ERRORS = "sentry.hive.authorization.errors"; + public static final String HIVE_SENTRY_MOCK_COMPILATION = "sentry.hive.mock.compilation"; + public static final String HIVE_SENTRY_MOCK_ERROR = "sentry.hive.mock.error"; + public static final String HIVE_SENTRY_PRIVILEGE_ERROR_MESSAGE = "No valid privileges"; + /** + * Property used to persist the role set in the session. This is not public for now. + */ + public static final String SENTRY_ACTIVE_ROLE_SET = "hive.sentry.active.role.set"; + + public static final String HIVE_SENTRY_SECURITY_COMMAND_WHITELIST = + "hive.sentry.security.command.whitelist"; + public static final String HIVE_SENTRY_SECURITY_COMMAND_WHITELIST_DEFAULT = + "set,reset,reload"; + + public static final String HIVE_SENTRY_SERDE_WHITELIST = "hive.sentry.serde.whitelist"; + public static final String HIVE_SENTRY_SERDE_WHITELIST_DEFAULT = "org.apache.hadoop.hive.serde2"; + + // Disable the serde Uri privileges by default for backward compatibilities. + public static final String HIVE_SENTRY_SERDE_URI_PRIVILIEGES_ENABLED = "hive.sentry.turn.on.serde.uri.privileges"; + public static final boolean HIVE_SENTRY_SERDE_URI_PRIVILIEGES_ENABLED_DEFAULT = false; + + public static final String HIVE_UDF_WHITE_LIST = + "concat,substr,substring,space,repeat,ascii,lpad,rpad,size,round,floor,sqrt,ceil," + + "ceiling,rand,abs,pmod,ln,log2,sin,asin,cos,acos,log10,log,exp,power,pow,sign,pi," + + "degrees,radians,atan,tan,e,conv,bin,hex,unhex,base64,unbase64,encode,decode,upper," + + "lower,ucase,lcase,trim,ltrim,rtrim,length,reverse,field,find_in_set,initcap,like," + + "rlike,regexp,regexp_replace,regexp_extract,parse_url,nvl,split,str_to_map,translate" + + ",positive,negative,day,dayofmonth,month,year,hour,minute,second,from_unixtime," + + "to_date,weekofyear,last_day,date_add,date_sub,datediff,add_months,get_json_object," + + "xpath_string,xpath_boolean,xpath_number,xpath_double,xpath_float,xpath_long," + + "xpath_int,xpath_short,xpath,+,-,*,/,%,div,&,|,^,~,current_database,isnull," + + "isnotnull,if,in,and,or,=,==,<=>,!=,<>,<,<=,>,>=,not,!,between,ewah_bitmap_and," + + "ewah_bitmap_or,ewah_bitmap_empty,boolean,tinyint,smallint,int,bigint,float,double," + + "string,date,timestamp,binary,decimal,varchar,char,max,min,sum,count,avg,std,stddev," + + "stddev_pop,stddev_samp,variance,var_pop,var_samp,covar_pop,covar_samp,corr," + + "histogram_numeric,percentile_approx,collect_set,collect_list,ngrams," + + "context_ngrams,ewah_bitmap,compute_stats,percentile," + + "array,assert_true,map,struct,named_struct,create_union,case,when,hash,coalesce," + + "index,in_file,instr,locate,elt,concat_ws,sort_array," + + "array_contains,sentences,map_keys,map_values,format_number,printf,greatest,least," + + "from_utc_timestamp,to_utc_timestamp,unix_timestamp,to_unix_timestamp,explode," + + "inline,json_tuple,parse_url_tuple,posexplode,stack,lead,lag,row_number,rank," + + "dense_rank,percent_rank,cume_dist,ntile,first_value,last_value,noop,noopwithmap," + + "noopstreaming,noopwithmapstreaming,windowingtablefunction,matchpath"; + + public static final String HIVE_UDF_BLACK_LIST = "reflect,reflect2,java_method"; + + /** + * Config setting definitions + */ + public static enum AuthzConfVars { + AUTHZ_PROVIDER("sentry.provider", + "org.apache.sentry.provider.common.HadoopGroupResourceAuthorizationProvider"), + AUTHZ_PROVIDER_RESOURCE("sentry.hive.provider.resource", ""), + AUTHZ_PROVIDER_BACKEND("sentry.hive.provider.backend", "org.apache.sentry.provider.file.SimpleFileProviderBackend"), + AUTHZ_POLICY_ENGINE("sentry.hive.policy.engine", "org.apache.sentry.policy.engine.common.CommonPolicyEngine"), + AUTHZ_POLICY_FILE_FORMATTER( + "sentry.hive.policy.file.formatter", + "org.apache.sentry.binding.hive.SentryIniPolicyFileFormatter"), + AUTHZ_SERVER_NAME("sentry.hive.server", ""), + AUTHZ_RESTRICT_DEFAULT_DB("sentry.hive.restrict.defaultDB", "false"), + SENTRY_TESTING_MODE("sentry.hive.testing.mode", "false"), + AUTHZ_ALLOW_HIVE_IMPERSONATION("sentry.hive.allow.hive.impersonation", "false"), + AUTHZ_ONFAILURE_HOOKS("sentry.hive.failure.hooks", ""), + AUTHZ_METASTORE_SERVICE_USERS("sentry.metastore.service.users", null), + AUTHZ_SYNC_ALTER_WITH_POLICY_STORE("sentry.hive.sync.alter", "true"), + AUTHZ_SYNC_CREATE_WITH_POLICY_STORE("sentry.hive.sync.create", "false"), + AUTHZ_SYNC_DROP_WITH_POLICY_STORE("sentry.hive.sync.drop", "true"), + + AUTHZ_PROVIDER_DEPRECATED("hive.sentry.provider", + "org.apache.sentry.provider.file.ResourceAuthorizationProvider"), + AUTHZ_PROVIDER_RESOURCE_DEPRECATED("hive.sentry.provider.resource", ""), + AUTHZ_SERVER_NAME_DEPRECATED("hive.sentry.server", ""), + AUTHZ_RESTRICT_DEFAULT_DB_DEPRECATED("hive.sentry.restrict.defaultDB", "false"), + SENTRY_TESTING_MODE_DEPRECATED("hive.sentry.testing.mode", "false"), + AUTHZ_ALLOW_HIVE_IMPERSONATION_DEPRECATED("hive.sentry.allow.hive.impersonation", "false"), + AUTHZ_ONFAILURE_HOOKS_DEPRECATED("hive.sentry.failure.hooks", ""); + + private final String varName; + private final String defaultVal; + + AuthzConfVars(String varName, String defaultVal) { + this.varName = varName; + this.defaultVal = defaultVal; + } + + public String getVar() { + return varName; + } + + public String getDefault() { + return defaultVal; + } + + public static String getDefault(String varName) { + for (AuthzConfVars oneVar : AuthzConfVars.values()) { + if(oneVar.getVar().equalsIgnoreCase(varName)) { + return oneVar.getDefault(); + } + } + return null; + } + } + + // map of current property names - > deprecated property names. + // The binding layer code should work if the deprecated property names are provided, + // as long as the new property names aren't also provided. Since the binding code + // only calls the new property names, we require a map from current names to deprecated + // names in order to check if the deprecated name of a property was set. + private static final Map<String, AuthzConfVars> currentToDeprecatedProps = + new HashMap<String, AuthzConfVars>(); + static { + currentToDeprecatedProps.put(AuthzConfVars.AUTHZ_PROVIDER.getVar(), AuthzConfVars.AUTHZ_PROVIDER_DEPRECATED); + currentToDeprecatedProps.put(AuthzConfVars.AUTHZ_PROVIDER_RESOURCE.getVar(), AuthzConfVars.AUTHZ_PROVIDER_RESOURCE_DEPRECATED); + currentToDeprecatedProps.put(AuthzConfVars.AUTHZ_SERVER_NAME.getVar(), AuthzConfVars.AUTHZ_SERVER_NAME_DEPRECATED); + currentToDeprecatedProps.put(AuthzConfVars.AUTHZ_RESTRICT_DEFAULT_DB.getVar(), AuthzConfVars.AUTHZ_RESTRICT_DEFAULT_DB_DEPRECATED); + currentToDeprecatedProps.put(AuthzConfVars.SENTRY_TESTING_MODE.getVar(), AuthzConfVars.SENTRY_TESTING_MODE_DEPRECATED); + currentToDeprecatedProps.put(AuthzConfVars.AUTHZ_ALLOW_HIVE_IMPERSONATION.getVar(), AuthzConfVars.AUTHZ_ALLOW_HIVE_IMPERSONATION_DEPRECATED); + currentToDeprecatedProps.put(AuthzConfVars.AUTHZ_ONFAILURE_HOOKS.getVar(), AuthzConfVars.AUTHZ_ONFAILURE_HOOKS_DEPRECATED); + }; + + private static final Logger LOG = LoggerFactory + .getLogger(HiveAuthzConf.class); + public static final String AUTHZ_SITE_FILE = "sentry-site.xml"; + private final String hiveAuthzSiteFile; + + public HiveAuthzConf(URL hiveAuthzSiteURL) { + super(); + LOG.info("DefaultFS: " + super.get("fs.defaultFS")); + addResource(hiveAuthzSiteURL); + applySystemProperties(); + LOG.info("DefaultFS: " + super.get("fs.defaultFS")); + this.hiveAuthzSiteFile = hiveAuthzSiteURL.toString(); + } + /** + * Apply system properties to this object if the property name is defined in ConfVars + * and the value is non-null and not an empty string. + */ + private void applySystemProperties() { + Map<String, String> systemProperties = getConfSystemProperties(); + for (Entry<String, String> systemProperty : systemProperties.entrySet()) { + this.set(systemProperty.getKey(), systemProperty.getValue()); + } + } + + /** + * This method returns a mapping from config variable name to its value for all config variables + * which have been set using System properties + */ + public static Map<String, String> getConfSystemProperties() { + Map<String, String> systemProperties = new HashMap<String, String>(); + + for (AuthzConfVars oneVar : AuthzConfVars.values()) { + String value = System.getProperty(oneVar.getVar()); + if (value != null && value.length() > 0) { + systemProperties.put(oneVar.getVar(), value); + } + } + return systemProperties; + } + + @Override + public String get(String varName) { + return get(varName, null); + } + + @Override + public String get(String varName, String defaultVal) { + String retVal = super.get(varName); + if (retVal == null) { + // check if the deprecated value is set here + if (currentToDeprecatedProps.containsKey(varName)) { + retVal = super.get(currentToDeprecatedProps.get(varName).getVar()); + } + if (retVal == null) { + retVal = AuthzConfVars.getDefault(varName); + } else { + LOG.warn("Using the deprecated config setting " + currentToDeprecatedProps.get(varName).getVar() + + " instead of " + varName); + } + } + if (retVal == null) { + retVal = defaultVal; + } + return retVal; + } + + public String getHiveAuthzSiteFile() { + return hiveAuthzSiteFile; + } + + /** + * Extract the authz config file path from given hive conf and load the authz config + * @param hiveConf + * @return + * @throws IllegalArgumentException + */ + public static HiveAuthzConf getAuthzConf(HiveConf hiveConf) + throws IllegalArgumentException { + boolean depreicatedConfigFile = false; + + String hiveAuthzConf = hiveConf.get(HiveAuthzConf.HIVE_SENTRY_CONF_URL); + if (hiveAuthzConf == null + || (hiveAuthzConf = hiveAuthzConf.trim()).isEmpty()) { + hiveAuthzConf = hiveConf.get(HiveAuthzConf.HIVE_ACCESS_CONF_URL); + depreicatedConfigFile = true; + } + + if (hiveAuthzConf == null + || (hiveAuthzConf = hiveAuthzConf.trim()).isEmpty()) { + throw new IllegalArgumentException("Configuration key " + + HiveAuthzConf.HIVE_SENTRY_CONF_URL + " value '" + hiveAuthzConf + + "' is invalid."); + } + + try { + return new HiveAuthzConf(new URL(hiveAuthzConf)); + } catch (MalformedURLException e) { + if (depreicatedConfigFile) { + throw new IllegalArgumentException("Configuration key " + + HiveAuthzConf.HIVE_ACCESS_CONF_URL + + " specifies a malformed URL '" + hiveAuthzConf + "'", e); + } else { + throw new IllegalArgumentException("Configuration key " + + HiveAuthzConf.HIVE_SENTRY_CONF_URL + + " specifies a malformed URL '" + hiveAuthzConf + "'", e); + } + } + } +} http://git-wip-us.apache.org/repos/asf/sentry/blob/5a96dcda/sentry-binding/sentry-binding-hive-follower/pom.xml ---------------------------------------------------------------------- diff --git a/sentry-binding/sentry-binding-hive-follower/pom.xml b/sentry-binding/sentry-binding-hive-follower/pom.xml new file mode 100644 index 0000000..bcfb417 --- /dev/null +++ b/sentry-binding/sentry-binding-hive-follower/pom.xml @@ -0,0 +1,71 @@ +<?xml version="1.0"?> +<!-- +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +--> +<project xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"; xmlns="http://maven.apache.org/POM/4.0.0"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";> + <modelVersion>4.0.0</modelVersion> + + <parent> + <groupId>org.apache.sentry</groupId> + <artifactId>sentry-binding</artifactId> + <version>1.8.0-SNAPSHOT</version> + </parent> + + <artifactId>sentry-binding-hive-follower</artifactId> + <name>Hive follower for Sentry</name> + + <properties> + <datanucleus-api-jdo.version>3.2.6</datanucleus-api-jdo.version> + <datanucleus-core.version>3.2.12</datanucleus-core.version> + <datanucleus-rdbms.version>3.2.12</datanucleus-rdbms.version> + </properties> + + <dependencies> + <dependency> + <groupId>org.datanucleus</groupId> + <artifactId>datanucleus-core</artifactId> + <version>${datanucleus-core.version}</version> + </dependency> + <dependency> + <groupId>org.datanucleus</groupId> + <artifactId>datanucleus-api-jdo</artifactId> + <version>${datanucleus-api-jdo.version}</version> + </dependency> + <dependency> + <groupId>org.datanucleus</groupId> + <artifactId>datanucleus-rdbms</artifactId> + <version>${datanucleus-rdbms.version}</version> + </dependency> + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.apache.hive</groupId> + <artifactId>hive-exec</artifactId> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.apache.hive.hcatalog</groupId> + <artifactId>hive-hcatalog-server-extensions</artifactId> + <version>${hive.version}</version> + <scope>compile</scope> + </dependency> + </dependencies> + +</project> http://git-wip-us.apache.org/repos/asf/sentry/blob/5a96dcda/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/JSONAlterPartitionMessage.java ---------------------------------------------------------------------- diff --git a/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/JSONAlterPartitionMessage.java b/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/JSONAlterPartitionMessage.java new file mode 100644 index 0000000..890186b --- /dev/null +++ b/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/JSONAlterPartitionMessage.java @@ -0,0 +1,78 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.sentry.binding.metastore.messaging.json; + +import org.apache.hive.hcatalog.messaging.AlterPartitionMessage; +import org.codehaus.jackson.annotate.JsonProperty; + +import java.util.List; + +/* +* This is only needed as corresponding class in Hive 1.1.0 does not have a default constructor + */ +public class JSONAlterPartitionMessage extends AlterPartitionMessage { + @JsonProperty + String server; + @JsonProperty + String servicePrincipal; + @JsonProperty + String db; + @JsonProperty + String table; + @JsonProperty + Long timestamp; + @JsonProperty + List<String> values; + + public JSONAlterPartitionMessage() {} + public JSONAlterPartitionMessage(String server, String servicePrincipal, String db, String table, List<String> values, Long timestamp) { + this.server = server; + this.servicePrincipal = servicePrincipal; + this.db = db; + this.table = table; + this.timestamp = timestamp; + this.values = values; + this.checkValid(); + } + + public String getServer() { + return this.server; + } + + public String getServicePrincipal() { + return this.servicePrincipal; + } + + public String getDB() { + return this.db; + } + + public Long getTimestamp() { + return this.timestamp; + } + + public String getTable() { + return this.table; + } + + public List<String> getValues() { + return this.values; + } + +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/sentry/blob/5a96dcda/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/JSONAlterTableMessage.java ---------------------------------------------------------------------- diff --git a/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/JSONAlterTableMessage.java b/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/JSONAlterTableMessage.java new file mode 100644 index 0000000..76211c3 --- /dev/null +++ b/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/JSONAlterTableMessage.java @@ -0,0 +1,68 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.sentry.binding.metastore.messaging.json; + +import org.apache.hive.hcatalog.messaging.AlterTableMessage; +import org.codehaus.jackson.annotate.JsonProperty; + +/** + * This class is required as this class does not have a default contructor in Hive 1.1.0 + */ +public class JSONAlterTableMessage extends AlterTableMessage { + @JsonProperty + String server; + @JsonProperty + String servicePrincipal; + @JsonProperty + String db; + @JsonProperty + String table; + @JsonProperty + Long timestamp; + + public JSONAlterTableMessage() {} + public JSONAlterTableMessage(String server, String servicePrincipal, String db, String table, Long timestamp) { + this.server = server; + this.servicePrincipal = servicePrincipal; + this.db = db; + this.table = table; + this.timestamp = timestamp; + this.checkValid(); + } + + public String getServer() { + return this.server; + } + + public String getServicePrincipal() { + return this.servicePrincipal; + } + + public String getDB() { + return this.db; + } + + public Long getTimestamp() { + return this.timestamp; + } + + public String getTable() { + return this.table; + } +} http://git-wip-us.apache.org/repos/asf/sentry/blob/5a96dcda/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONAddPartitionMessage.java ---------------------------------------------------------------------- diff --git a/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONAddPartitionMessage.java b/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONAddPartitionMessage.java new file mode 100644 index 0000000..c0c469c --- /dev/null +++ b/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONAddPartitionMessage.java @@ -0,0 +1,49 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.sentry.binding.metastore.messaging.json; + +import org.apache.hive.hcatalog.messaging.json.JSONAddPartitionMessage; +import org.codehaus.jackson.annotate.JsonProperty; + +import java.util.List; +import java.util.Map; + +public class SentryJSONAddPartitionMessage extends JSONAddPartitionMessage { + @JsonProperty + List<String> locations; + + public SentryJSONAddPartitionMessage() { + } + + public SentryJSONAddPartitionMessage(String server, String servicePrincipal, String db, String table, + List<Map<String, String>> partitions, Long timestamp, List<String> locations) { + super(server, servicePrincipal, db, table, partitions, timestamp); + this.locations = locations; + } + + public List<String> getLocations() { + return locations; + } + + @Override + public String toString() { + return SentryJSONMessageDeserializer.serialize(this); + } + +} http://git-wip-us.apache.org/repos/asf/sentry/blob/5a96dcda/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONAlterPartitionMessage.java ---------------------------------------------------------------------- diff --git a/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONAlterPartitionMessage.java b/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONAlterPartitionMessage.java new file mode 100644 index 0000000..99eb67a --- /dev/null +++ b/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONAlterPartitionMessage.java @@ -0,0 +1,53 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.sentry.binding.metastore.messaging.json; + +import org.codehaus.jackson.annotate.JsonProperty; + +import java.util.List; + +public class SentryJSONAlterPartitionMessage extends JSONAlterPartitionMessage{ + @JsonProperty + String location; + @JsonProperty + String oldLocation; + + public SentryJSONAlterPartitionMessage() { + } + + public SentryJSONAlterPartitionMessage(String server, String servicePrincipal, String db, String table, + List<String> values, Long timestamp, String oldlocation, String newLocation) { + super(server, servicePrincipal, db, table, values, timestamp); + this.location = newLocation; + this.oldLocation = oldlocation; + } + + public String getLocation() { + return location; + } + + public String getOldLocation() { + return oldLocation; + } + + @Override + public String toString() { + return SentryJSONMessageDeserializer.serialize(this); + } +} http://git-wip-us.apache.org/repos/asf/sentry/blob/5a96dcda/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONAlterTableMessage.java ---------------------------------------------------------------------- diff --git a/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONAlterTableMessage.java b/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONAlterTableMessage.java new file mode 100644 index 0000000..6e59e25 --- /dev/null +++ b/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONAlterTableMessage.java @@ -0,0 +1,50 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.sentry.binding.metastore.messaging.json; + +import org.codehaus.jackson.annotate.JsonProperty; + +public class SentryJSONAlterTableMessage extends JSONAlterTableMessage { + @JsonProperty + String location; //newLocation + @JsonProperty + String oldLocation; + + public SentryJSONAlterTableMessage() { + } + + public SentryJSONAlterTableMessage(String server, String servicePrincipal, String db, String table, + Long timestamp, String oldLocation, String location) { + super(server, servicePrincipal, db, table, timestamp); + this.location = location; + this.oldLocation = oldLocation; + } + + public String getLocation() { + return location; + } + public String getOldLocation() { + return oldLocation; + } + + @Override + public String toString() { + return SentryJSONMessageDeserializer.serialize(this); + } +} http://git-wip-us.apache.org/repos/asf/sentry/blob/5a96dcda/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONCreateDatabaseMessage.java ---------------------------------------------------------------------- diff --git a/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONCreateDatabaseMessage.java b/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONCreateDatabaseMessage.java new file mode 100644 index 0000000..ba19cbe --- /dev/null +++ b/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONCreateDatabaseMessage.java @@ -0,0 +1,44 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.sentry.binding.metastore.messaging.json; + +import org.apache.hive.hcatalog.messaging.json.JSONCreateDatabaseMessage; +import org.codehaus.jackson.annotate.JsonProperty; + +public class SentryJSONCreateDatabaseMessage extends JSONCreateDatabaseMessage { + @JsonProperty + String location; + + public SentryJSONCreateDatabaseMessage() { + } + + public SentryJSONCreateDatabaseMessage(String server, String servicePrincipal, String db, Long timestamp, String location) { + super(server, servicePrincipal, db, timestamp); + this.location = location; + } + + public String getLocation() { + return location; + } + + @Override + public String toString() { + return SentryJSONMessageDeserializer.serialize(this); + } +} http://git-wip-us.apache.org/repos/asf/sentry/blob/5a96dcda/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONCreateTableMessage.java ---------------------------------------------------------------------- diff --git a/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONCreateTableMessage.java b/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONCreateTableMessage.java new file mode 100644 index 0000000..57d11d2 --- /dev/null +++ b/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONCreateTableMessage.java @@ -0,0 +1,45 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.sentry.binding.metastore.messaging.json; + +import org.apache.hive.hcatalog.messaging.json.JSONCreateTableMessage; +import org.codehaus.jackson.annotate.JsonProperty; + +public class SentryJSONCreateTableMessage extends JSONCreateTableMessage { + @JsonProperty + String location; + + public SentryJSONCreateTableMessage() { + } + + public SentryJSONCreateTableMessage(String server, String servicePrincipal, String db, String table, Long timestamp, String location) { + super(server, servicePrincipal, db, table, timestamp); + this.location = location; + } + + public String getLocation() { + return location; + } + + @Override + public String toString() { + return SentryJSONMessageDeserializer.serialize(this); + } + +} http://git-wip-us.apache.org/repos/asf/sentry/blob/5a96dcda/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONDropDatabaseMessage.java ---------------------------------------------------------------------- diff --git a/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONDropDatabaseMessage.java b/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONDropDatabaseMessage.java new file mode 100644 index 0000000..05f83f7 --- /dev/null +++ b/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONDropDatabaseMessage.java @@ -0,0 +1,44 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.sentry.binding.metastore.messaging.json; + +import org.apache.hive.hcatalog.messaging.json.JSONDropDatabaseMessage; +import org.codehaus.jackson.annotate.JsonProperty; + +public class SentryJSONDropDatabaseMessage extends JSONDropDatabaseMessage{ + @JsonProperty + String location; + + public SentryJSONDropDatabaseMessage() { + } + + public SentryJSONDropDatabaseMessage(String server, String servicePrincipal, String db, Long timestamp, String location) { + super(server, servicePrincipal, db, timestamp); + this.location = location; + } + + public String getLocation() { + return location; + } + + @Override + public String toString() { + return SentryJSONMessageDeserializer.serialize(this); + } +} http://git-wip-us.apache.org/repos/asf/sentry/blob/5a96dcda/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONDropPartitionMessage.java ---------------------------------------------------------------------- diff --git a/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONDropPartitionMessage.java b/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONDropPartitionMessage.java new file mode 100644 index 0000000..2ab61f7 --- /dev/null +++ b/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONDropPartitionMessage.java @@ -0,0 +1,49 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.sentry.binding.metastore.messaging.json; + +import org.apache.hive.hcatalog.messaging.json.JSONDropPartitionMessage; +import org.codehaus.jackson.annotate.JsonProperty; + +import java.util.List; +import java.util.Map; + +public class SentryJSONDropPartitionMessage extends JSONDropPartitionMessage { + @JsonProperty + String location; + + public SentryJSONDropPartitionMessage() { + } + + public SentryJSONDropPartitionMessage(String server, String servicePrincipal, String db, String table, + List<Map<String, String>> partitions, Long timestamp, String location) { + super(server, servicePrincipal, db, table, partitions, timestamp); + this.location = location; + } + + public String getLocation() { + return location; + } + + @Override + public String toString() { + return SentryJSONMessageDeserializer.serialize(this); + } + +} http://git-wip-us.apache.org/repos/asf/sentry/blob/5a96dcda/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONDropTableMessage.java ---------------------------------------------------------------------- diff --git a/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONDropTableMessage.java b/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONDropTableMessage.java new file mode 100644 index 0000000..7005776 --- /dev/null +++ b/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONDropTableMessage.java @@ -0,0 +1,45 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.sentry.binding.metastore.messaging.json; + +import org.apache.hive.hcatalog.messaging.json.JSONDropTableMessage; +import org.codehaus.jackson.annotate.JsonProperty; + + +public class SentryJSONDropTableMessage extends JSONDropTableMessage { + @JsonProperty + String location; + + public SentryJSONDropTableMessage() { + } + + public SentryJSONDropTableMessage(String server, String servicePrincipal, String db, String table, Long timestamp, String location) { + super(server, servicePrincipal, db, table, timestamp); + this.location = location; + } + + public String getLocation() { + return location; + } + + @Override + public String toString() { + return SentryJSONMessageDeserializer.serialize(this); + } +} http://git-wip-us.apache.org/repos/asf/sentry/blob/5a96dcda/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONMessageDeserializer.java ---------------------------------------------------------------------- diff --git a/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONMessageDeserializer.java b/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONMessageDeserializer.java new file mode 100644 index 0000000..b645c45 --- /dev/null +++ b/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONMessageDeserializer.java @@ -0,0 +1,110 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.sentry.binding.metastore.messaging.json; + +import org.apache.hive.hcatalog.messaging.*; +import org.codehaus.jackson.map.DeserializationConfig; +import org.codehaus.jackson.map.ObjectMapper; + +public class SentryJSONMessageDeserializer extends MessageDeserializer { + static ObjectMapper mapper = new ObjectMapper(); + + public SentryJSONMessageDeserializer() { + } + + /** + * Method to de-serialize CreateDatabaseMessage instance. + */ + public SentryJSONCreateDatabaseMessage getCreateDatabaseMessage(String messageBody) { + try { + return (SentryJSONCreateDatabaseMessage)mapper.readValue(messageBody, SentryJSONCreateDatabaseMessage.class); + } catch (Exception var3) { + throw new IllegalArgumentException("Could not construct SentryJSONCreateDatabaseMessage.", var3); + } + } + + public SentryJSONDropDatabaseMessage getDropDatabaseMessage(String messageBody) { + try { + return (SentryJSONDropDatabaseMessage)mapper.readValue(messageBody, SentryJSONDropDatabaseMessage.class); + } catch (Exception var3) { + throw new IllegalArgumentException("Could not construct SentryJSONDropDatabaseMessage.", var3); + } + } + + public SentryJSONCreateTableMessage getCreateTableMessage(String messageBody) { + try { + return (SentryJSONCreateTableMessage)mapper.readValue(messageBody, SentryJSONCreateTableMessage.class); + } catch (Exception var3) { + throw new IllegalArgumentException("Could not construct SentryJSONCreateTableMessage.", var3); + } + } + + public SentryJSONAlterTableMessage getAlterTableMessage(String messageBody) { + try { + return (SentryJSONAlterTableMessage)mapper.readValue(messageBody, SentryJSONAlterTableMessage.class); + } catch (Exception var3) { + throw new IllegalArgumentException("Could not construct SentryJSONAlterTableMessage.", var3); + } + } + + public SentryJSONDropTableMessage getDropTableMessage(String messageBody) { + try { + return (SentryJSONDropTableMessage)mapper.readValue(messageBody, SentryJSONDropTableMessage.class); + } catch (Exception var3) { + throw new IllegalArgumentException("Could not construct SentryJSONDropTableMessage.", var3); + } + } + + public SentryJSONAddPartitionMessage getAddPartitionMessage(String messageBody) { + try { + return (SentryJSONAddPartitionMessage)mapper.readValue(messageBody, SentryJSONAddPartitionMessage.class); + } catch (Exception var3) { + throw new IllegalArgumentException("Could not construct SentryJSONAddPartitionMessage.", var3); + } + } + + public SentryJSONAlterPartitionMessage getAlterPartitionMessage(String messageBody) { + try { + return (SentryJSONAlterPartitionMessage)mapper.readValue(messageBody, SentryJSONAlterPartitionMessage.class); + } catch (Exception var3) { + throw new IllegalArgumentException("Could not construct SentryJSONAlterPartitionMessage.", var3); + } + } + + public SentryJSONDropPartitionMessage getDropPartitionMessage(String messageBody) { + try { + return (SentryJSONDropPartitionMessage)mapper.readValue(messageBody, SentryJSONDropPartitionMessage.class); + } catch (Exception var3) { + throw new IllegalArgumentException("Could not construct SentryJSONDropPartitionMessage.", var3); + } + } + + static { + mapper.configure(DeserializationConfig.Feature.FAIL_ON_UNKNOWN_PROPERTIES, false); + } + + public static String serialize(Object object) { + try { + return mapper.writeValueAsString(object); + } + catch (Exception exception) { + throw new IllegalArgumentException("Could not serialize: ", exception); + } + } +} http://git-wip-us.apache.org/repos/asf/sentry/blob/5a96dcda/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONMessageFactory.java ---------------------------------------------------------------------- diff --git a/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONMessageFactory.java b/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONMessageFactory.java new file mode 100644 index 0000000..00e7db8 --- /dev/null +++ b/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONMessageFactory.java @@ -0,0 +1,177 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sentry.binding.metastore.messaging.json; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hive.common.classification.InterfaceAudience; +import org.apache.hadoop.hive.common.classification.InterfaceStability; +import org.apache.hadoop.hive.metastore.api.Database; +import org.apache.hadoop.hive.metastore.api.FieldSchema; +import org.apache.hadoop.hive.metastore.api.Partition; +import org.apache.hadoop.hive.metastore.api.Table; +import org.apache.hadoop.hive.metastore.partition.spec.PartitionSpecProxy; +import org.apache.hive.hcatalog.messaging.*; + +import java.util.*; + +public class SentryJSONMessageFactory extends MessageFactory { + private static final Log LOG = LogFactory.getLog(SentryJSONMessageFactory.class.getName()); + private static SentryJSONMessageDeserializer deserializer = new SentryJSONMessageDeserializer(); + public SentryJSONMessageFactory() { + LOG.info("Using SentryJSONMessageFactory for building Notification log messages "); + + } + public MessageDeserializer getDeserializer() { + return deserializer; + } + + public String getVersion() { + return "0.1"; + } + + public String getMessageFormat() { + return "json"; + } + + public SentryJSONCreateDatabaseMessage buildCreateDatabaseMessage(Database db) { + return new SentryJSONCreateDatabaseMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, db.getName(), + Long.valueOf(this.now()), db.getLocationUri()); + } + public SentryJSONDropDatabaseMessage buildDropDatabaseMessage(Database db) { + return new SentryJSONDropDatabaseMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, db.getName(), + Long.valueOf(this.now()), db.getLocationUri()); + } + + public SentryJSONCreateTableMessage buildCreateTableMessage(Table table) { + return new SentryJSONCreateTableMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, table.getDbName(), + table.getTableName(), Long.valueOf(this.now()), table.getSd().getLocation()); + } + + public SentryJSONAlterTableMessage buildAlterTableMessage(Table before, Table after) { + return new SentryJSONAlterTableMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, before.getDbName(), + before.getTableName(), Long.valueOf(this.now()), before.getSd().getLocation(), after.getSd().getLocation()); + } + + public SentryJSONDropTableMessage buildDropTableMessage(Table table) { + return new SentryJSONDropTableMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, table.getDbName(), + table.getTableName(), Long.valueOf(this.now()), table.getSd().getLocation()); + } + + public SentryJSONAddPartitionMessage buildAddPartitionMessage(Table table, List<Partition> partitions) { + return new SentryJSONAddPartitionMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, table.getDbName(), + table.getTableName(), getPartitionKeyValues(table, partitions), Long.valueOf(this.now()), + getPartitionLocations(partitions)); + } + + private List<String> getPartitionLocations(List<Partition> partitions) { + List<String> paths = new ArrayList<String>(); + for(Partition partition:partitions) { + paths.add(partition.getSd().getLocation()); + } + return paths; + } + + //TODO: Not sure what is this used for. Need to investigate + private List<String> getPartitionLocations(PartitionSpecProxy partitionSpec) { + Iterator<Partition> iterator = partitionSpec.getPartitionIterator(); + List<String> locations = new ArrayList<String>(); + while(iterator.hasNext()) { + locations.add(iterator.next().getSd().getLocation()); + } + return locations; + } + + @InterfaceAudience.LimitedPrivate({"Hive"}) + @InterfaceStability.Evolving + public SentryJSONAddPartitionMessage buildAddPartitionMessage(Table table, PartitionSpecProxy partitionSpec) { + return new SentryJSONAddPartitionMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, table.getDbName(), + table.getTableName(), getPartitionKeyValues(table, partitionSpec), Long.valueOf(this.now()), + getPartitionLocations(partitionSpec)); + } + + public SentryJSONAlterPartitionMessage buildAlterPartitionMessage(Partition before, Partition after) { + /* + f (partitionEvent.getOldPartition() != null) { + oldLoc = partitionEvent.getOldPartition().getSd().getLocation(); + } + if (partitionEvent.getNewPartition() != null) { + newLoc = partitionEvent.getNewPartition().getSd().getLocation(); + } + + if ((oldLoc != null) && (newLoc != null) && (!oldLoc.equals(newLoc))) { + String authzObj = + partitionEvent.getOldPartition().getDbName() + "." + + partitionEvent.getOldPartition().getTableName(); + for (SentryMetastoreListenerPlugin plugin : sentryPlugins) { + plugin.renameAuthzObject(authzObj, oldLoc, + authzObj, newLoc); + } + } + * */ + return new SentryJSONAlterPartitionMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, before.getDbName(), + before.getTableName(), before.getValues(), Long.valueOf(this.now()), before.getSd().getLocation(), + after.getSd().getLocation()); + } + + public SentryJSONDropPartitionMessage buildDropPartitionMessage(Table table, Partition partition) { + return new SentryJSONDropPartitionMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, partition.getDbName(), + partition.getTableName(), Arrays.asList(getPartitionKeyValues(table, partition)), + Long.valueOf(this.now()), partition.getSd().getLocation()); + } + + private static Map<String, String> getPartitionKeyValues(Table table, Partition partition) { + LinkedHashMap partitionKeys = new LinkedHashMap(); + + for(int i = 0; i < table.getPartitionKeysSize(); ++i) { + partitionKeys.put(((FieldSchema)table.getPartitionKeys().get(i)).getName(), partition.getValues().get(i)); + } + + return partitionKeys; + } + + private static List<Map<String, String>> getPartitionKeyValues(Table table, List<Partition> partitions) { + ArrayList partitionList = new ArrayList(partitions.size()); + Iterator i$ = partitions.iterator(); + + while(i$.hasNext()) { + Partition partition = (Partition)i$.next(); + partitionList.add(getPartitionKeyValues(table, partition)); + } + + return partitionList; + } + + @InterfaceAudience.LimitedPrivate({"Hive"}) + @InterfaceStability.Evolving + private static List<Map<String, String>> getPartitionKeyValues(Table table, PartitionSpecProxy partitionSpec) { + ArrayList partitionList = new ArrayList(); + PartitionSpecProxy.PartitionIterator iterator = partitionSpec.getPartitionIterator(); + + while(iterator.hasNext()) { + Partition partition = (Partition)iterator.next(); + partitionList.add(getPartitionKeyValues(table, partition)); + } + + return partitionList; + } + //This is private in parent class + private long now() { + return System.currentTimeMillis() / 1000L; + } +} http://git-wip-us.apache.org/repos/asf/sentry/blob/5a96dcda/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/JSONAlterPartitionMessage.java ---------------------------------------------------------------------- diff --git a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/JSONAlterPartitionMessage.java b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/JSONAlterPartitionMessage.java deleted file mode 100644 index 890186b..0000000 --- a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/JSONAlterPartitionMessage.java +++ /dev/null @@ -1,78 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.sentry.binding.metastore.messaging.json; - -import org.apache.hive.hcatalog.messaging.AlterPartitionMessage; -import org.codehaus.jackson.annotate.JsonProperty; - -import java.util.List; - -/* -* This is only needed as corresponding class in Hive 1.1.0 does not have a default constructor - */ -public class JSONAlterPartitionMessage extends AlterPartitionMessage { - @JsonProperty - String server; - @JsonProperty - String servicePrincipal; - @JsonProperty - String db; - @JsonProperty - String table; - @JsonProperty - Long timestamp; - @JsonProperty - List<String> values; - - public JSONAlterPartitionMessage() {} - public JSONAlterPartitionMessage(String server, String servicePrincipal, String db, String table, List<String> values, Long timestamp) { - this.server = server; - this.servicePrincipal = servicePrincipal; - this.db = db; - this.table = table; - this.timestamp = timestamp; - this.values = values; - this.checkValid(); - } - - public String getServer() { - return this.server; - } - - public String getServicePrincipal() { - return this.servicePrincipal; - } - - public String getDB() { - return this.db; - } - - public Long getTimestamp() { - return this.timestamp; - } - - public String getTable() { - return this.table; - } - - public List<String> getValues() { - return this.values; - } - -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/sentry/blob/5a96dcda/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/JSONAlterTableMessage.java ---------------------------------------------------------------------- diff --git a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/JSONAlterTableMessage.java b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/JSONAlterTableMessage.java deleted file mode 100644 index 76211c3..0000000 --- a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/JSONAlterTableMessage.java +++ /dev/null @@ -1,68 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.sentry.binding.metastore.messaging.json; - -import org.apache.hive.hcatalog.messaging.AlterTableMessage; -import org.codehaus.jackson.annotate.JsonProperty; - -/** - * This class is required as this class does not have a default contructor in Hive 1.1.0 - */ -public class JSONAlterTableMessage extends AlterTableMessage { - @JsonProperty - String server; - @JsonProperty - String servicePrincipal; - @JsonProperty - String db; - @JsonProperty - String table; - @JsonProperty - Long timestamp; - - public JSONAlterTableMessage() {} - public JSONAlterTableMessage(String server, String servicePrincipal, String db, String table, Long timestamp) { - this.server = server; - this.servicePrincipal = servicePrincipal; - this.db = db; - this.table = table; - this.timestamp = timestamp; - this.checkValid(); - } - - public String getServer() { - return this.server; - } - - public String getServicePrincipal() { - return this.servicePrincipal; - } - - public String getDB() { - return this.db; - } - - public Long getTimestamp() { - return this.timestamp; - } - - public String getTable() { - return this.table; - } -} http://git-wip-us.apache.org/repos/asf/sentry/blob/5a96dcda/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONAddPartitionMessage.java ---------------------------------------------------------------------- diff --git a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONAddPartitionMessage.java b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONAddPartitionMessage.java deleted file mode 100644 index c0c469c..0000000 --- a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONAddPartitionMessage.java +++ /dev/null @@ -1,49 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.sentry.binding.metastore.messaging.json; - -import org.apache.hive.hcatalog.messaging.json.JSONAddPartitionMessage; -import org.codehaus.jackson.annotate.JsonProperty; - -import java.util.List; -import java.util.Map; - -public class SentryJSONAddPartitionMessage extends JSONAddPartitionMessage { - @JsonProperty - List<String> locations; - - public SentryJSONAddPartitionMessage() { - } - - public SentryJSONAddPartitionMessage(String server, String servicePrincipal, String db, String table, - List<Map<String, String>> partitions, Long timestamp, List<String> locations) { - super(server, servicePrincipal, db, table, partitions, timestamp); - this.locations = locations; - } - - public List<String> getLocations() { - return locations; - } - - @Override - public String toString() { - return SentryJSONMessageDeserializer.serialize(this); - } - -}
