Repository: sentry Updated Branches: refs/heads/sentry-ha-redesign 39c473c1f -> 2d5ed9984
Sentry-1321: Implement HMSFollower in Sentry service which reads the NotificationLog entries (Sravya Tirukkovalur, Reviewed by: Hao Hao) Change-Id: Ic63cb094bef12cf23f14c16cc0580ba8e9f84318 Project: http://git-wip-us.apache.org/repos/asf/sentry/repo Commit: http://git-wip-us.apache.org/repos/asf/sentry/commit/2d5ed998 Tree: http://git-wip-us.apache.org/repos/asf/sentry/tree/2d5ed998 Diff: http://git-wip-us.apache.org/repos/asf/sentry/diff/2d5ed998 Branch: refs/heads/sentry-ha-redesign Commit: 2d5ed99849343e3adf4c394f7e0abc0079b29452 Parents: 39c473c Author: Sravya Tirukkovalur <[email protected]> Authored: Mon Aug 15 15:54:40 2016 -0700 Committer: Sravya Tirukkovalur <[email protected]> Committed: Mon Aug 15 15:54:40 2016 -0700 ---------------------------------------------------------------------- sentry-binding/pom.xml | 3 + .../sentry-binding-hive-common/pom.xml | 5 + .../sentry/binding/hive/conf/HiveAuthzConf.java | 269 ------------------- .../json/JSONAlterPartitionMessage.java | 78 ------ .../messaging/json/JSONAlterTableMessage.java | 68 ----- .../json/SentryJSONAddPartitionMessage.java | 49 ---- .../json/SentryJSONAlterPartitionMessage.java | 53 ---- .../json/SentryJSONAlterTableMessage.java | 50 ---- .../json/SentryJSONCreateDatabaseMessage.java | 44 --- .../json/SentryJSONCreateTableMessage.java | 45 ---- .../json/SentryJSONDropDatabaseMessage.java | 44 --- .../json/SentryJSONDropPartitionMessage.java | 49 ---- .../json/SentryJSONDropTableMessage.java | 45 ---- .../json/SentryJSONMessageDeserializer.java | 110 -------- .../json/SentryJSONMessageFactory.java | 177 ------------ .../exception/SentryInvalidInputException.java | 3 + sentry-provider/sentry-provider-db/pom.xml | 31 +++ .../service/thrift/SentryKerberosContext.java | 10 +- .../sentry/service/thrift/SentryService.java | 22 +- .../TestDbPrivilegeCleanupOnDrop.java | 25 +- .../AbstractTestWithStaticConfiguration.java | 7 +- .../e2e/hive/hiveserver/HiveServerFactory.java | 4 + 22 files changed, 91 insertions(+), 1100 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sentry/blob/2d5ed998/sentry-binding/pom.xml ---------------------------------------------------------------------- diff --git a/sentry-binding/pom.xml b/sentry-binding/pom.xml index 30bca14..48ae245 100644 --- a/sentry-binding/pom.xml +++ b/sentry-binding/pom.xml @@ -34,6 +34,9 @@ limitations under the License. <module>sentry-binding-hive-common</module> <module>sentry-binding-solr</module> <module>sentry-binding-sqoop</module> + <module>sentry-binding-hive-follower</module> + <module>sentry-binding-hive-conf</module> + </modules> <profiles> http://git-wip-us.apache.org/repos/asf/sentry/blob/2d5ed998/sentry-binding/sentry-binding-hive-common/pom.xml ---------------------------------------------------------------------- diff --git a/sentry-binding/sentry-binding-hive-common/pom.xml b/sentry-binding/sentry-binding-hive-common/pom.xml index 18b422d..d38a494 100644 --- a/sentry-binding/sentry-binding-hive-common/pom.xml +++ b/sentry-binding/sentry-binding-hive-common/pom.xml @@ -78,6 +78,11 @@ limitations under the License. <scope>provided</scope> </dependency> <dependency> + <groupId>org.apache.sentry</groupId> + <artifactId>sentry-binding-hive-conf</artifactId> + <version>1.8.0-SNAPSHOT</version> + </dependency> + <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-client</artifactId> <version>${hadoop.version}</version> http://git-wip-us.apache.org/repos/asf/sentry/blob/2d5ed998/sentry-binding/sentry-binding-hive-common/src/main/java/org/apache/sentry/binding/hive/conf/HiveAuthzConf.java ---------------------------------------------------------------------- diff --git a/sentry-binding/sentry-binding-hive-common/src/main/java/org/apache/sentry/binding/hive/conf/HiveAuthzConf.java b/sentry-binding/sentry-binding-hive-common/src/main/java/org/apache/sentry/binding/hive/conf/HiveAuthzConf.java deleted file mode 100644 index ad19b37..0000000 --- a/sentry-binding/sentry-binding-hive-common/src/main/java/org/apache/sentry/binding/hive/conf/HiveAuthzConf.java +++ /dev/null @@ -1,269 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.sentry.binding.hive.conf; - -import java.net.MalformedURLException; -import java.net.URL; -import java.util.HashMap; -import java.util.Map; -import java.util.Map.Entry; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hive.conf.HiveConf; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - - -public class HiveAuthzConf extends Configuration { - - /** - * Configuration key used in hive-site.xml to point at sentry-site.xml - */ - public static final String HIVE_ACCESS_CONF_URL = "hive.access.conf.url"; - public static final String HIVE_SENTRY_CONF_URL = "hive.sentry.conf.url"; - public static final String HIVE_ACCESS_SUBJECT_NAME = "hive.access.subject.name"; - public static final String HIVE_SENTRY_SUBJECT_NAME = "hive.sentry.subject.name"; - public static final String HIVE_SENTRY_AUTH_ERRORS = "sentry.hive.authorization.errors"; - public static final String HIVE_SENTRY_MOCK_COMPILATION = "sentry.hive.mock.compilation"; - public static final String HIVE_SENTRY_MOCK_ERROR = "sentry.hive.mock.error"; - public static final String HIVE_SENTRY_PRIVILEGE_ERROR_MESSAGE = "No valid privileges"; - /** - * Property used to persist the role set in the session. This is not public for now. - */ - public static final String SENTRY_ACTIVE_ROLE_SET = "hive.sentry.active.role.set"; - - public static final String HIVE_SENTRY_SECURITY_COMMAND_WHITELIST = - "hive.sentry.security.command.whitelist"; - public static final String HIVE_SENTRY_SECURITY_COMMAND_WHITELIST_DEFAULT = - "set,reset,reload"; - - public static final String HIVE_SENTRY_SERDE_WHITELIST = "hive.sentry.serde.whitelist"; - public static final String HIVE_SENTRY_SERDE_WHITELIST_DEFAULT = "org.apache.hadoop.hive.serde2"; - - // Disable the serde Uri privileges by default for backward compatibilities. - public static final String HIVE_SENTRY_SERDE_URI_PRIVILIEGES_ENABLED = "hive.sentry.turn.on.serde.uri.privileges"; - public static final boolean HIVE_SENTRY_SERDE_URI_PRIVILIEGES_ENABLED_DEFAULT = false; - - public static final String HIVE_UDF_WHITE_LIST = - "concat,substr,substring,space,repeat,ascii,lpad,rpad,size,round,floor,sqrt,ceil," + - "ceiling,rand,abs,pmod,ln,log2,sin,asin,cos,acos,log10,log,exp,power,pow,sign,pi," + - "degrees,radians,atan,tan,e,conv,bin,hex,unhex,base64,unbase64,encode,decode,upper," + - "lower,ucase,lcase,trim,ltrim,rtrim,length,reverse,field,find_in_set,initcap,like," + - "rlike,regexp,regexp_replace,regexp_extract,parse_url,nvl,split,str_to_map,translate" + - ",positive,negative,day,dayofmonth,month,year,hour,minute,second,from_unixtime," + - "to_date,weekofyear,last_day,date_add,date_sub,datediff,add_months,get_json_object," + - "xpath_string,xpath_boolean,xpath_number,xpath_double,xpath_float,xpath_long," + - "xpath_int,xpath_short,xpath,+,-,*,/,%,div,&,|,^,~,current_database,isnull," + - "isnotnull,if,in,and,or,=,==,<=>,!=,<>,<,<=,>,>=,not,!,between,ewah_bitmap_and," + - "ewah_bitmap_or,ewah_bitmap_empty,boolean,tinyint,smallint,int,bigint,float,double," + - "string,date,timestamp,binary,decimal,varchar,char,max,min,sum,count,avg,std,stddev," + - "stddev_pop,stddev_samp,variance,var_pop,var_samp,covar_pop,covar_samp,corr," + - "histogram_numeric,percentile_approx,collect_set,collect_list,ngrams," + - "context_ngrams,ewah_bitmap,compute_stats,percentile," + - "array,assert_true,map,struct,named_struct,create_union,case,when,hash,coalesce," + - "index,in_file,instr,locate,elt,concat_ws,sort_array," + - "array_contains,sentences,map_keys,map_values,format_number,printf,greatest,least," + - "from_utc_timestamp,to_utc_timestamp,unix_timestamp,to_unix_timestamp,explode," + - "inline,json_tuple,parse_url_tuple,posexplode,stack,lead,lag,row_number,rank," + - "dense_rank,percent_rank,cume_dist,ntile,first_value,last_value,noop,noopwithmap," + - "noopstreaming,noopwithmapstreaming,windowingtablefunction,matchpath"; - - public static final String HIVE_UDF_BLACK_LIST = "reflect,reflect2,java_method"; - - /** - * Config setting definitions - */ - public static enum AuthzConfVars { - AUTHZ_PROVIDER("sentry.provider", - "org.apache.sentry.provider.common.HadoopGroupResourceAuthorizationProvider"), - AUTHZ_PROVIDER_RESOURCE("sentry.hive.provider.resource", ""), - AUTHZ_PROVIDER_BACKEND("sentry.hive.provider.backend", "org.apache.sentry.provider.file.SimpleFileProviderBackend"), - AUTHZ_POLICY_ENGINE("sentry.hive.policy.engine", "org.apache.sentry.policy.engine.common.CommonPolicyEngine"), - AUTHZ_POLICY_FILE_FORMATTER( - "sentry.hive.policy.file.formatter", - "org.apache.sentry.binding.hive.SentryIniPolicyFileFormatter"), - AUTHZ_SERVER_NAME("sentry.hive.server", ""), - AUTHZ_RESTRICT_DEFAULT_DB("sentry.hive.restrict.defaultDB", "false"), - SENTRY_TESTING_MODE("sentry.hive.testing.mode", "false"), - AUTHZ_ALLOW_HIVE_IMPERSONATION("sentry.hive.allow.hive.impersonation", "false"), - AUTHZ_ONFAILURE_HOOKS("sentry.hive.failure.hooks", ""), - AUTHZ_METASTORE_SERVICE_USERS("sentry.metastore.service.users", null), - AUTHZ_SYNC_ALTER_WITH_POLICY_STORE("sentry.hive.sync.alter", "true"), - AUTHZ_SYNC_CREATE_WITH_POLICY_STORE("sentry.hive.sync.create", "false"), - AUTHZ_SYNC_DROP_WITH_POLICY_STORE("sentry.hive.sync.drop", "true"), - - AUTHZ_PROVIDER_DEPRECATED("hive.sentry.provider", - "org.apache.sentry.provider.file.ResourceAuthorizationProvider"), - AUTHZ_PROVIDER_RESOURCE_DEPRECATED("hive.sentry.provider.resource", ""), - AUTHZ_SERVER_NAME_DEPRECATED("hive.sentry.server", ""), - AUTHZ_RESTRICT_DEFAULT_DB_DEPRECATED("hive.sentry.restrict.defaultDB", "false"), - SENTRY_TESTING_MODE_DEPRECATED("hive.sentry.testing.mode", "false"), - AUTHZ_ALLOW_HIVE_IMPERSONATION_DEPRECATED("hive.sentry.allow.hive.impersonation", "false"), - AUTHZ_ONFAILURE_HOOKS_DEPRECATED("hive.sentry.failure.hooks", ""); - - private final String varName; - private final String defaultVal; - - AuthzConfVars(String varName, String defaultVal) { - this.varName = varName; - this.defaultVal = defaultVal; - } - - public String getVar() { - return varName; - } - - public String getDefault() { - return defaultVal; - } - - public static String getDefault(String varName) { - for (AuthzConfVars oneVar : AuthzConfVars.values()) { - if(oneVar.getVar().equalsIgnoreCase(varName)) { - return oneVar.getDefault(); - } - } - return null; - } - } - - // map of current property names - > deprecated property names. - // The binding layer code should work if the deprecated property names are provided, - // as long as the new property names aren't also provided. Since the binding code - // only calls the new property names, we require a map from current names to deprecated - // names in order to check if the deprecated name of a property was set. - private static final Map<String, AuthzConfVars> currentToDeprecatedProps = - new HashMap<String, AuthzConfVars>(); - static { - currentToDeprecatedProps.put(AuthzConfVars.AUTHZ_PROVIDER.getVar(), AuthzConfVars.AUTHZ_PROVIDER_DEPRECATED); - currentToDeprecatedProps.put(AuthzConfVars.AUTHZ_PROVIDER_RESOURCE.getVar(), AuthzConfVars.AUTHZ_PROVIDER_RESOURCE_DEPRECATED); - currentToDeprecatedProps.put(AuthzConfVars.AUTHZ_SERVER_NAME.getVar(), AuthzConfVars.AUTHZ_SERVER_NAME_DEPRECATED); - currentToDeprecatedProps.put(AuthzConfVars.AUTHZ_RESTRICT_DEFAULT_DB.getVar(), AuthzConfVars.AUTHZ_RESTRICT_DEFAULT_DB_DEPRECATED); - currentToDeprecatedProps.put(AuthzConfVars.SENTRY_TESTING_MODE.getVar(), AuthzConfVars.SENTRY_TESTING_MODE_DEPRECATED); - currentToDeprecatedProps.put(AuthzConfVars.AUTHZ_ALLOW_HIVE_IMPERSONATION.getVar(), AuthzConfVars.AUTHZ_ALLOW_HIVE_IMPERSONATION_DEPRECATED); - currentToDeprecatedProps.put(AuthzConfVars.AUTHZ_ONFAILURE_HOOKS.getVar(), AuthzConfVars.AUTHZ_ONFAILURE_HOOKS_DEPRECATED); - }; - - private static final Logger LOG = LoggerFactory - .getLogger(HiveAuthzConf.class); - public static final String AUTHZ_SITE_FILE = "sentry-site.xml"; - private final String hiveAuthzSiteFile; - - public HiveAuthzConf(URL hiveAuthzSiteURL) { - super(); - LOG.info("DefaultFS: " + super.get("fs.defaultFS")); - addResource(hiveAuthzSiteURL); - applySystemProperties(); - LOG.info("DefaultFS: " + super.get("fs.defaultFS")); - this.hiveAuthzSiteFile = hiveAuthzSiteURL.toString(); - } - /** - * Apply system properties to this object if the property name is defined in ConfVars - * and the value is non-null and not an empty string. - */ - private void applySystemProperties() { - Map<String, String> systemProperties = getConfSystemProperties(); - for (Entry<String, String> systemProperty : systemProperties.entrySet()) { - this.set(systemProperty.getKey(), systemProperty.getValue()); - } - } - - /** - * This method returns a mapping from config variable name to its value for all config variables - * which have been set using System properties - */ - public static Map<String, String> getConfSystemProperties() { - Map<String, String> systemProperties = new HashMap<String, String>(); - - for (AuthzConfVars oneVar : AuthzConfVars.values()) { - String value = System.getProperty(oneVar.getVar()); - if (value != null && value.length() > 0) { - systemProperties.put(oneVar.getVar(), value); - } - } - return systemProperties; - } - - @Override - public String get(String varName) { - return get(varName, null); - } - - @Override - public String get(String varName, String defaultVal) { - String retVal = super.get(varName); - if (retVal == null) { - // check if the deprecated value is set here - if (currentToDeprecatedProps.containsKey(varName)) { - retVal = super.get(currentToDeprecatedProps.get(varName).getVar()); - } - if (retVal == null) { - retVal = AuthzConfVars.getDefault(varName); - } else { - LOG.warn("Using the deprecated config setting " + currentToDeprecatedProps.get(varName).getVar() + - " instead of " + varName); - } - } - if (retVal == null) { - retVal = defaultVal; - } - return retVal; - } - - public String getHiveAuthzSiteFile() { - return hiveAuthzSiteFile; - } - - /** - * Extract the authz config file path from given hive conf and load the authz config - * @param hiveConf - * @return - * @throws IllegalArgumentException - */ - public static HiveAuthzConf getAuthzConf(HiveConf hiveConf) - throws IllegalArgumentException { - boolean depreicatedConfigFile = false; - - String hiveAuthzConf = hiveConf.get(HiveAuthzConf.HIVE_SENTRY_CONF_URL); - if (hiveAuthzConf == null - || (hiveAuthzConf = hiveAuthzConf.trim()).isEmpty()) { - hiveAuthzConf = hiveConf.get(HiveAuthzConf.HIVE_ACCESS_CONF_URL); - depreicatedConfigFile = true; - } - - if (hiveAuthzConf == null - || (hiveAuthzConf = hiveAuthzConf.trim()).isEmpty()) { - throw new IllegalArgumentException("Configuration key " - + HiveAuthzConf.HIVE_SENTRY_CONF_URL + " value '" + hiveAuthzConf - + "' is invalid."); - } - - try { - return new HiveAuthzConf(new URL(hiveAuthzConf)); - } catch (MalformedURLException e) { - if (depreicatedConfigFile) { - throw new IllegalArgumentException("Configuration key " - + HiveAuthzConf.HIVE_ACCESS_CONF_URL - + " specifies a malformed URL '" + hiveAuthzConf + "'", e); - } else { - throw new IllegalArgumentException("Configuration key " - + HiveAuthzConf.HIVE_SENTRY_CONF_URL - + " specifies a malformed URL '" + hiveAuthzConf + "'", e); - } - } - } -} http://git-wip-us.apache.org/repos/asf/sentry/blob/2d5ed998/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/JSONAlterPartitionMessage.java ---------------------------------------------------------------------- diff --git a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/JSONAlterPartitionMessage.java b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/JSONAlterPartitionMessage.java deleted file mode 100644 index 890186b..0000000 --- a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/JSONAlterPartitionMessage.java +++ /dev/null @@ -1,78 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.sentry.binding.metastore.messaging.json; - -import org.apache.hive.hcatalog.messaging.AlterPartitionMessage; -import org.codehaus.jackson.annotate.JsonProperty; - -import java.util.List; - -/* -* This is only needed as corresponding class in Hive 1.1.0 does not have a default constructor - */ -public class JSONAlterPartitionMessage extends AlterPartitionMessage { - @JsonProperty - String server; - @JsonProperty - String servicePrincipal; - @JsonProperty - String db; - @JsonProperty - String table; - @JsonProperty - Long timestamp; - @JsonProperty - List<String> values; - - public JSONAlterPartitionMessage() {} - public JSONAlterPartitionMessage(String server, String servicePrincipal, String db, String table, List<String> values, Long timestamp) { - this.server = server; - this.servicePrincipal = servicePrincipal; - this.db = db; - this.table = table; - this.timestamp = timestamp; - this.values = values; - this.checkValid(); - } - - public String getServer() { - return this.server; - } - - public String getServicePrincipal() { - return this.servicePrincipal; - } - - public String getDB() { - return this.db; - } - - public Long getTimestamp() { - return this.timestamp; - } - - public String getTable() { - return this.table; - } - - public List<String> getValues() { - return this.values; - } - -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/sentry/blob/2d5ed998/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/JSONAlterTableMessage.java ---------------------------------------------------------------------- diff --git a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/JSONAlterTableMessage.java b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/JSONAlterTableMessage.java deleted file mode 100644 index 76211c3..0000000 --- a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/JSONAlterTableMessage.java +++ /dev/null @@ -1,68 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.sentry.binding.metastore.messaging.json; - -import org.apache.hive.hcatalog.messaging.AlterTableMessage; -import org.codehaus.jackson.annotate.JsonProperty; - -/** - * This class is required as this class does not have a default contructor in Hive 1.1.0 - */ -public class JSONAlterTableMessage extends AlterTableMessage { - @JsonProperty - String server; - @JsonProperty - String servicePrincipal; - @JsonProperty - String db; - @JsonProperty - String table; - @JsonProperty - Long timestamp; - - public JSONAlterTableMessage() {} - public JSONAlterTableMessage(String server, String servicePrincipal, String db, String table, Long timestamp) { - this.server = server; - this.servicePrincipal = servicePrincipal; - this.db = db; - this.table = table; - this.timestamp = timestamp; - this.checkValid(); - } - - public String getServer() { - return this.server; - } - - public String getServicePrincipal() { - return this.servicePrincipal; - } - - public String getDB() { - return this.db; - } - - public Long getTimestamp() { - return this.timestamp; - } - - public String getTable() { - return this.table; - } -} http://git-wip-us.apache.org/repos/asf/sentry/blob/2d5ed998/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONAddPartitionMessage.java ---------------------------------------------------------------------- diff --git a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONAddPartitionMessage.java b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONAddPartitionMessage.java deleted file mode 100644 index c0c469c..0000000 --- a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONAddPartitionMessage.java +++ /dev/null @@ -1,49 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.sentry.binding.metastore.messaging.json; - -import org.apache.hive.hcatalog.messaging.json.JSONAddPartitionMessage; -import org.codehaus.jackson.annotate.JsonProperty; - -import java.util.List; -import java.util.Map; - -public class SentryJSONAddPartitionMessage extends JSONAddPartitionMessage { - @JsonProperty - List<String> locations; - - public SentryJSONAddPartitionMessage() { - } - - public SentryJSONAddPartitionMessage(String server, String servicePrincipal, String db, String table, - List<Map<String, String>> partitions, Long timestamp, List<String> locations) { - super(server, servicePrincipal, db, table, partitions, timestamp); - this.locations = locations; - } - - public List<String> getLocations() { - return locations; - } - - @Override - public String toString() { - return SentryJSONMessageDeserializer.serialize(this); - } - -} http://git-wip-us.apache.org/repos/asf/sentry/blob/2d5ed998/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONAlterPartitionMessage.java ---------------------------------------------------------------------- diff --git a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONAlterPartitionMessage.java b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONAlterPartitionMessage.java deleted file mode 100644 index 99eb67a..0000000 --- a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONAlterPartitionMessage.java +++ /dev/null @@ -1,53 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.sentry.binding.metastore.messaging.json; - -import org.codehaus.jackson.annotate.JsonProperty; - -import java.util.List; - -public class SentryJSONAlterPartitionMessage extends JSONAlterPartitionMessage{ - @JsonProperty - String location; - @JsonProperty - String oldLocation; - - public SentryJSONAlterPartitionMessage() { - } - - public SentryJSONAlterPartitionMessage(String server, String servicePrincipal, String db, String table, - List<String> values, Long timestamp, String oldlocation, String newLocation) { - super(server, servicePrincipal, db, table, values, timestamp); - this.location = newLocation; - this.oldLocation = oldlocation; - } - - public String getLocation() { - return location; - } - - public String getOldLocation() { - return oldLocation; - } - - @Override - public String toString() { - return SentryJSONMessageDeserializer.serialize(this); - } -} http://git-wip-us.apache.org/repos/asf/sentry/blob/2d5ed998/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONAlterTableMessage.java ---------------------------------------------------------------------- diff --git a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONAlterTableMessage.java b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONAlterTableMessage.java deleted file mode 100644 index 6e59e25..0000000 --- a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONAlterTableMessage.java +++ /dev/null @@ -1,50 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.sentry.binding.metastore.messaging.json; - -import org.codehaus.jackson.annotate.JsonProperty; - -public class SentryJSONAlterTableMessage extends JSONAlterTableMessage { - @JsonProperty - String location; //newLocation - @JsonProperty - String oldLocation; - - public SentryJSONAlterTableMessage() { - } - - public SentryJSONAlterTableMessage(String server, String servicePrincipal, String db, String table, - Long timestamp, String oldLocation, String location) { - super(server, servicePrincipal, db, table, timestamp); - this.location = location; - this.oldLocation = oldLocation; - } - - public String getLocation() { - return location; - } - public String getOldLocation() { - return oldLocation; - } - - @Override - public String toString() { - return SentryJSONMessageDeserializer.serialize(this); - } -} http://git-wip-us.apache.org/repos/asf/sentry/blob/2d5ed998/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONCreateDatabaseMessage.java ---------------------------------------------------------------------- diff --git a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONCreateDatabaseMessage.java b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONCreateDatabaseMessage.java deleted file mode 100644 index ba19cbe..0000000 --- a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONCreateDatabaseMessage.java +++ /dev/null @@ -1,44 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.sentry.binding.metastore.messaging.json; - -import org.apache.hive.hcatalog.messaging.json.JSONCreateDatabaseMessage; -import org.codehaus.jackson.annotate.JsonProperty; - -public class SentryJSONCreateDatabaseMessage extends JSONCreateDatabaseMessage { - @JsonProperty - String location; - - public SentryJSONCreateDatabaseMessage() { - } - - public SentryJSONCreateDatabaseMessage(String server, String servicePrincipal, String db, Long timestamp, String location) { - super(server, servicePrincipal, db, timestamp); - this.location = location; - } - - public String getLocation() { - return location; - } - - @Override - public String toString() { - return SentryJSONMessageDeserializer.serialize(this); - } -} http://git-wip-us.apache.org/repos/asf/sentry/blob/2d5ed998/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONCreateTableMessage.java ---------------------------------------------------------------------- diff --git a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONCreateTableMessage.java b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONCreateTableMessage.java deleted file mode 100644 index 57d11d2..0000000 --- a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONCreateTableMessage.java +++ /dev/null @@ -1,45 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.sentry.binding.metastore.messaging.json; - -import org.apache.hive.hcatalog.messaging.json.JSONCreateTableMessage; -import org.codehaus.jackson.annotate.JsonProperty; - -public class SentryJSONCreateTableMessage extends JSONCreateTableMessage { - @JsonProperty - String location; - - public SentryJSONCreateTableMessage() { - } - - public SentryJSONCreateTableMessage(String server, String servicePrincipal, String db, String table, Long timestamp, String location) { - super(server, servicePrincipal, db, table, timestamp); - this.location = location; - } - - public String getLocation() { - return location; - } - - @Override - public String toString() { - return SentryJSONMessageDeserializer.serialize(this); - } - -} http://git-wip-us.apache.org/repos/asf/sentry/blob/2d5ed998/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONDropDatabaseMessage.java ---------------------------------------------------------------------- diff --git a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONDropDatabaseMessage.java b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONDropDatabaseMessage.java deleted file mode 100644 index 05f83f7..0000000 --- a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONDropDatabaseMessage.java +++ /dev/null @@ -1,44 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.sentry.binding.metastore.messaging.json; - -import org.apache.hive.hcatalog.messaging.json.JSONDropDatabaseMessage; -import org.codehaus.jackson.annotate.JsonProperty; - -public class SentryJSONDropDatabaseMessage extends JSONDropDatabaseMessage{ - @JsonProperty - String location; - - public SentryJSONDropDatabaseMessage() { - } - - public SentryJSONDropDatabaseMessage(String server, String servicePrincipal, String db, Long timestamp, String location) { - super(server, servicePrincipal, db, timestamp); - this.location = location; - } - - public String getLocation() { - return location; - } - - @Override - public String toString() { - return SentryJSONMessageDeserializer.serialize(this); - } -} http://git-wip-us.apache.org/repos/asf/sentry/blob/2d5ed998/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONDropPartitionMessage.java ---------------------------------------------------------------------- diff --git a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONDropPartitionMessage.java b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONDropPartitionMessage.java deleted file mode 100644 index 2ab61f7..0000000 --- a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONDropPartitionMessage.java +++ /dev/null @@ -1,49 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.sentry.binding.metastore.messaging.json; - -import org.apache.hive.hcatalog.messaging.json.JSONDropPartitionMessage; -import org.codehaus.jackson.annotate.JsonProperty; - -import java.util.List; -import java.util.Map; - -public class SentryJSONDropPartitionMessage extends JSONDropPartitionMessage { - @JsonProperty - String location; - - public SentryJSONDropPartitionMessage() { - } - - public SentryJSONDropPartitionMessage(String server, String servicePrincipal, String db, String table, - List<Map<String, String>> partitions, Long timestamp, String location) { - super(server, servicePrincipal, db, table, partitions, timestamp); - this.location = location; - } - - public String getLocation() { - return location; - } - - @Override - public String toString() { - return SentryJSONMessageDeserializer.serialize(this); - } - -} http://git-wip-us.apache.org/repos/asf/sentry/blob/2d5ed998/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONDropTableMessage.java ---------------------------------------------------------------------- diff --git a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONDropTableMessage.java b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONDropTableMessage.java deleted file mode 100644 index 7005776..0000000 --- a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONDropTableMessage.java +++ /dev/null @@ -1,45 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.sentry.binding.metastore.messaging.json; - -import org.apache.hive.hcatalog.messaging.json.JSONDropTableMessage; -import org.codehaus.jackson.annotate.JsonProperty; - - -public class SentryJSONDropTableMessage extends JSONDropTableMessage { - @JsonProperty - String location; - - public SentryJSONDropTableMessage() { - } - - public SentryJSONDropTableMessage(String server, String servicePrincipal, String db, String table, Long timestamp, String location) { - super(server, servicePrincipal, db, table, timestamp); - this.location = location; - } - - public String getLocation() { - return location; - } - - @Override - public String toString() { - return SentryJSONMessageDeserializer.serialize(this); - } -} http://git-wip-us.apache.org/repos/asf/sentry/blob/2d5ed998/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONMessageDeserializer.java ---------------------------------------------------------------------- diff --git a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONMessageDeserializer.java b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONMessageDeserializer.java deleted file mode 100644 index b645c45..0000000 --- a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONMessageDeserializer.java +++ /dev/null @@ -1,110 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.sentry.binding.metastore.messaging.json; - -import org.apache.hive.hcatalog.messaging.*; -import org.codehaus.jackson.map.DeserializationConfig; -import org.codehaus.jackson.map.ObjectMapper; - -public class SentryJSONMessageDeserializer extends MessageDeserializer { - static ObjectMapper mapper = new ObjectMapper(); - - public SentryJSONMessageDeserializer() { - } - - /** - * Method to de-serialize CreateDatabaseMessage instance. - */ - public SentryJSONCreateDatabaseMessage getCreateDatabaseMessage(String messageBody) { - try { - return (SentryJSONCreateDatabaseMessage)mapper.readValue(messageBody, SentryJSONCreateDatabaseMessage.class); - } catch (Exception var3) { - throw new IllegalArgumentException("Could not construct SentryJSONCreateDatabaseMessage.", var3); - } - } - - public SentryJSONDropDatabaseMessage getDropDatabaseMessage(String messageBody) { - try { - return (SentryJSONDropDatabaseMessage)mapper.readValue(messageBody, SentryJSONDropDatabaseMessage.class); - } catch (Exception var3) { - throw new IllegalArgumentException("Could not construct SentryJSONDropDatabaseMessage.", var3); - } - } - - public SentryJSONCreateTableMessage getCreateTableMessage(String messageBody) { - try { - return (SentryJSONCreateTableMessage)mapper.readValue(messageBody, SentryJSONCreateTableMessage.class); - } catch (Exception var3) { - throw new IllegalArgumentException("Could not construct SentryJSONCreateTableMessage.", var3); - } - } - - public SentryJSONAlterTableMessage getAlterTableMessage(String messageBody) { - try { - return (SentryJSONAlterTableMessage)mapper.readValue(messageBody, SentryJSONAlterTableMessage.class); - } catch (Exception var3) { - throw new IllegalArgumentException("Could not construct SentryJSONAlterTableMessage.", var3); - } - } - - public SentryJSONDropTableMessage getDropTableMessage(String messageBody) { - try { - return (SentryJSONDropTableMessage)mapper.readValue(messageBody, SentryJSONDropTableMessage.class); - } catch (Exception var3) { - throw new IllegalArgumentException("Could not construct SentryJSONDropTableMessage.", var3); - } - } - - public SentryJSONAddPartitionMessage getAddPartitionMessage(String messageBody) { - try { - return (SentryJSONAddPartitionMessage)mapper.readValue(messageBody, SentryJSONAddPartitionMessage.class); - } catch (Exception var3) { - throw new IllegalArgumentException("Could not construct SentryJSONAddPartitionMessage.", var3); - } - } - - public SentryJSONAlterPartitionMessage getAlterPartitionMessage(String messageBody) { - try { - return (SentryJSONAlterPartitionMessage)mapper.readValue(messageBody, SentryJSONAlterPartitionMessage.class); - } catch (Exception var3) { - throw new IllegalArgumentException("Could not construct SentryJSONAlterPartitionMessage.", var3); - } - } - - public SentryJSONDropPartitionMessage getDropPartitionMessage(String messageBody) { - try { - return (SentryJSONDropPartitionMessage)mapper.readValue(messageBody, SentryJSONDropPartitionMessage.class); - } catch (Exception var3) { - throw new IllegalArgumentException("Could not construct SentryJSONDropPartitionMessage.", var3); - } - } - - static { - mapper.configure(DeserializationConfig.Feature.FAIL_ON_UNKNOWN_PROPERTIES, false); - } - - public static String serialize(Object object) { - try { - return mapper.writeValueAsString(object); - } - catch (Exception exception) { - throw new IllegalArgumentException("Could not serialize: ", exception); - } - } -} http://git-wip-us.apache.org/repos/asf/sentry/blob/2d5ed998/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONMessageFactory.java ---------------------------------------------------------------------- diff --git a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONMessageFactory.java b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONMessageFactory.java deleted file mode 100644 index 00e7db8..0000000 --- a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONMessageFactory.java +++ /dev/null @@ -1,177 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.sentry.binding.metastore.messaging.json; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.hive.common.classification.InterfaceAudience; -import org.apache.hadoop.hive.common.classification.InterfaceStability; -import org.apache.hadoop.hive.metastore.api.Database; -import org.apache.hadoop.hive.metastore.api.FieldSchema; -import org.apache.hadoop.hive.metastore.api.Partition; -import org.apache.hadoop.hive.metastore.api.Table; -import org.apache.hadoop.hive.metastore.partition.spec.PartitionSpecProxy; -import org.apache.hive.hcatalog.messaging.*; - -import java.util.*; - -public class SentryJSONMessageFactory extends MessageFactory { - private static final Log LOG = LogFactory.getLog(SentryJSONMessageFactory.class.getName()); - private static SentryJSONMessageDeserializer deserializer = new SentryJSONMessageDeserializer(); - public SentryJSONMessageFactory() { - LOG.info("Using SentryJSONMessageFactory for building Notification log messages "); - - } - public MessageDeserializer getDeserializer() { - return deserializer; - } - - public String getVersion() { - return "0.1"; - } - - public String getMessageFormat() { - return "json"; - } - - public SentryJSONCreateDatabaseMessage buildCreateDatabaseMessage(Database db) { - return new SentryJSONCreateDatabaseMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, db.getName(), - Long.valueOf(this.now()), db.getLocationUri()); - } - public SentryJSONDropDatabaseMessage buildDropDatabaseMessage(Database db) { - return new SentryJSONDropDatabaseMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, db.getName(), - Long.valueOf(this.now()), db.getLocationUri()); - } - - public SentryJSONCreateTableMessage buildCreateTableMessage(Table table) { - return new SentryJSONCreateTableMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, table.getDbName(), - table.getTableName(), Long.valueOf(this.now()), table.getSd().getLocation()); - } - - public SentryJSONAlterTableMessage buildAlterTableMessage(Table before, Table after) { - return new SentryJSONAlterTableMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, before.getDbName(), - before.getTableName(), Long.valueOf(this.now()), before.getSd().getLocation(), after.getSd().getLocation()); - } - - public SentryJSONDropTableMessage buildDropTableMessage(Table table) { - return new SentryJSONDropTableMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, table.getDbName(), - table.getTableName(), Long.valueOf(this.now()), table.getSd().getLocation()); - } - - public SentryJSONAddPartitionMessage buildAddPartitionMessage(Table table, List<Partition> partitions) { - return new SentryJSONAddPartitionMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, table.getDbName(), - table.getTableName(), getPartitionKeyValues(table, partitions), Long.valueOf(this.now()), - getPartitionLocations(partitions)); - } - - private List<String> getPartitionLocations(List<Partition> partitions) { - List<String> paths = new ArrayList<String>(); - for(Partition partition:partitions) { - paths.add(partition.getSd().getLocation()); - } - return paths; - } - - //TODO: Not sure what is this used for. Need to investigate - private List<String> getPartitionLocations(PartitionSpecProxy partitionSpec) { - Iterator<Partition> iterator = partitionSpec.getPartitionIterator(); - List<String> locations = new ArrayList<String>(); - while(iterator.hasNext()) { - locations.add(iterator.next().getSd().getLocation()); - } - return locations; - } - - @InterfaceAudience.LimitedPrivate({"Hive"}) - @InterfaceStability.Evolving - public SentryJSONAddPartitionMessage buildAddPartitionMessage(Table table, PartitionSpecProxy partitionSpec) { - return new SentryJSONAddPartitionMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, table.getDbName(), - table.getTableName(), getPartitionKeyValues(table, partitionSpec), Long.valueOf(this.now()), - getPartitionLocations(partitionSpec)); - } - - public SentryJSONAlterPartitionMessage buildAlterPartitionMessage(Partition before, Partition after) { - /* - f (partitionEvent.getOldPartition() != null) { - oldLoc = partitionEvent.getOldPartition().getSd().getLocation(); - } - if (partitionEvent.getNewPartition() != null) { - newLoc = partitionEvent.getNewPartition().getSd().getLocation(); - } - - if ((oldLoc != null) && (newLoc != null) && (!oldLoc.equals(newLoc))) { - String authzObj = - partitionEvent.getOldPartition().getDbName() + "." - + partitionEvent.getOldPartition().getTableName(); - for (SentryMetastoreListenerPlugin plugin : sentryPlugins) { - plugin.renameAuthzObject(authzObj, oldLoc, - authzObj, newLoc); - } - } - * */ - return new SentryJSONAlterPartitionMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, before.getDbName(), - before.getTableName(), before.getValues(), Long.valueOf(this.now()), before.getSd().getLocation(), - after.getSd().getLocation()); - } - - public SentryJSONDropPartitionMessage buildDropPartitionMessage(Table table, Partition partition) { - return new SentryJSONDropPartitionMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, partition.getDbName(), - partition.getTableName(), Arrays.asList(getPartitionKeyValues(table, partition)), - Long.valueOf(this.now()), partition.getSd().getLocation()); - } - - private static Map<String, String> getPartitionKeyValues(Table table, Partition partition) { - LinkedHashMap partitionKeys = new LinkedHashMap(); - - for(int i = 0; i < table.getPartitionKeysSize(); ++i) { - partitionKeys.put(((FieldSchema)table.getPartitionKeys().get(i)).getName(), partition.getValues().get(i)); - } - - return partitionKeys; - } - - private static List<Map<String, String>> getPartitionKeyValues(Table table, List<Partition> partitions) { - ArrayList partitionList = new ArrayList(partitions.size()); - Iterator i$ = partitions.iterator(); - - while(i$.hasNext()) { - Partition partition = (Partition)i$.next(); - partitionList.add(getPartitionKeyValues(table, partition)); - } - - return partitionList; - } - - @InterfaceAudience.LimitedPrivate({"Hive"}) - @InterfaceStability.Evolving - private static List<Map<String, String>> getPartitionKeyValues(Table table, PartitionSpecProxy partitionSpec) { - ArrayList partitionList = new ArrayList(); - PartitionSpecProxy.PartitionIterator iterator = partitionSpec.getPartitionIterator(); - - while(iterator.hasNext()) { - Partition partition = (Partition)iterator.next(); - partitionList.add(getPartitionKeyValues(table, partition)); - } - - return partitionList; - } - //This is private in parent class - private long now() { - return System.currentTimeMillis() / 1000L; - } -} http://git-wip-us.apache.org/repos/asf/sentry/blob/2d5ed998/sentry-core/sentry-core-common/src/main/java/org/apache/sentry/core/common/exception/SentryInvalidInputException.java ---------------------------------------------------------------------- diff --git a/sentry-core/sentry-core-common/src/main/java/org/apache/sentry/core/common/exception/SentryInvalidInputException.java b/sentry-core/sentry-core-common/src/main/java/org/apache/sentry/core/common/exception/SentryInvalidInputException.java index 903eddc..c36f166 100644 --- a/sentry-core/sentry-core-common/src/main/java/org/apache/sentry/core/common/exception/SentryInvalidInputException.java +++ b/sentry-core/sentry-core-common/src/main/java/org/apache/sentry/core/common/exception/SentryInvalidInputException.java @@ -25,4 +25,7 @@ public class SentryInvalidInputException extends SentryUserException { public SentryInvalidInputException(String msg, String reason) { super(msg, reason); } + public SentryInvalidInputException(String msg, Throwable cause) { + super(msg, cause); + } } http://git-wip-us.apache.org/repos/asf/sentry/blob/2d5ed998/sentry-provider/sentry-provider-db/pom.xml ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/pom.xml b/sentry-provider/sentry-provider-db/pom.xml index b8143ff..1699286 100644 --- a/sentry-provider/sentry-provider-db/pom.xml +++ b/sentry-provider/sentry-provider-db/pom.xml @@ -107,6 +107,16 @@ limitations under the License. <artifactId>sentry-policy-engine</artifactId> </dependency> <dependency> + <groupId>org.apache.sentry</groupId> + <artifactId>sentry-binding-hive-conf</artifactId> + <version>1.8.0-SNAPSHOT</version> + </dependency> + <dependency> + <groupId>org.apache.sentry</groupId> + <artifactId>sentry-binding-hive-follower</artifactId> + <version>1.8.0-SNAPSHOT</version> + </dependency> + <dependency> <groupId>org.apache.hive</groupId> <artifactId>hive-shims</artifactId> <scope>provided</scope> @@ -116,6 +126,10 @@ limitations under the License. <artifactId>hive-beeline</artifactId> </dependency> <dependency> + <groupId>org.apache.hive.hcatalog</groupId> + <artifactId>hive-hcatalog-server-extensions</artifactId> + </dependency> + <dependency> <groupId>org.apache.thrift</groupId> <artifactId>libfb303</artifactId> </dependency> @@ -177,6 +191,23 @@ limitations under the License. <groupId>org.apache.commons</groupId> <artifactId>commons-pool2</artifactId> </dependency> + <dependency> + <groupId>org.apache.hive</groupId> + <artifactId>hive-common</artifactId> + </dependency> + <dependency> + <groupId>org.apache.hive</groupId> + <artifactId>hive-metastore</artifactId> + </dependency> + <dependency> + <groupId>org.apache.hive</groupId> + <artifactId>hive-metastore</artifactId> + </dependency> + <dependency> + <groupId>org.apache.hive.hcatalog</groupId> + <artifactId>hive-hcatalog-server-extensions</artifactId> + <version>RELEASE</version> + </dependency> </dependencies> <build> http://git-wip-us.apache.org/repos/asf/sentry/blob/2d5ed998/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryKerberosContext.java ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryKerberosContext.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryKerberosContext.java index f54f161..8d78d1d 100644 --- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryKerberosContext.java +++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryKerberosContext.java @@ -45,13 +45,17 @@ public class SentryKerberosContext implements Runnable { @Deprecated private boolean shutDownRenewer = false; - public SentryKerberosContext(String principal, String keyTab, boolean autoRenewTicket) + public SentryKerberosContext(String principal, String keyTab, boolean server) throws LoginException { subject = new Subject(false, Sets.newHashSet(new KerberosPrincipal(principal)), new HashSet<Object>(), new HashSet<Object>()); - kerberosConfig = KerberosConfiguration.createClientConfig(principal, new File(keyTab)); + if(server) { + kerberosConfig = KerberosConfiguration.createServerConfig(principal, new File(keyTab)); + } else { + kerberosConfig = KerberosConfiguration.createClientConfig(principal, new File(keyTab)); + } loginWithNewContext(); - if (autoRenewTicket) { + if (!server) { startRenewerThread(); } } http://git-wip-us.apache.org/repos/asf/sentry/blob/2d5ed998/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryService.java ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryService.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryService.java index e93e5b4..ddcb90c 100644 --- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryService.java +++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryService.java @@ -28,12 +28,7 @@ import java.security.PrivilegedExceptionAction; import java.util.ArrayList; import java.util.EventListener; import java.util.List; -import java.util.concurrent.Callable; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; -import java.util.concurrent.ThreadFactory; +import java.util.concurrent.*; import javax.security.auth.Subject; @@ -90,6 +85,7 @@ public class SentryService implements Callable { private final String[] principalParts; private final String keytab; private final ExecutorService serviceExecutor; + private ScheduledExecutorService hmsFollowerExecutor = null; private Future serviceStatus; private TServer thriftServer; private Status status; @@ -158,6 +154,14 @@ public class SentryService implements Callable { conf.set(SentryConstants.CURRENT_INCARNATION_ID_KEY, this.act.getIncarnationId()); webServerPort = conf.getInt(ServerConfig.SENTRY_WEB_PORT, ServerConfig.SENTRY_WEB_PORT_DEFAULT); + //TODO: Enable only if Hive is using Sentry? + try { + hmsFollowerExecutor = Executors.newScheduledThreadPool(1); + hmsFollowerExecutor.scheduleAtFixedRate(new HMSFollower(conf), 60000, 500, TimeUnit.MILLISECONDS); + }catch(Exception e) { + //TODO: Handle + LOGGER.error("Could not start HMSFollower"); + } status = Status.NOT_STARTED; } @@ -167,8 +171,7 @@ public class SentryService implements Callable { try { status = Status.STARTED; if (kerberos) { - Boolean autoRenewTicket = conf.getBoolean(ServerConfig.SENTRY_KERBEROS_TGT_AUTORENEW, ServerConfig.SENTRY_KERBEROS_TGT_AUTORENEW_DEFAULT); - kerberosContext = new SentryKerberosContext(principal, keytab, autoRenewTicket); + kerberosContext = new SentryKerberosContext(principal, keytab, true); Subject.doAs(kerberosContext.getSubject(), new PrivilegedExceptionAction<Void>() { @Override public Void run() throws Exception { @@ -319,6 +322,9 @@ public class SentryService implements Callable { } else { LOGGER.info("Sentry web service is already stopped..."); } + if(hmsFollowerExecutor != null) { + hmsFollowerExecutor.shutdown(); + } if (exception != null) { exception.ifExceptionThrow(); } http://git-wip-us.apache.org/repos/asf/sentry/blob/2d5ed998/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/dbprovider/TestDbPrivilegeCleanupOnDrop.java ---------------------------------------------------------------------- diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/dbprovider/TestDbPrivilegeCleanupOnDrop.java b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/dbprovider/TestDbPrivilegeCleanupOnDrop.java index 439b9de..69fc8d2 100644 --- a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/dbprovider/TestDbPrivilegeCleanupOnDrop.java +++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/dbprovider/TestDbPrivilegeCleanupOnDrop.java @@ -31,14 +31,16 @@ import java.sql.Statement; import java.util.ArrayList; import java.util.List; +import org.apache.sentry.service.thrift.HMSFollower; import org.apache.sentry.tests.e2e.hive.AbstractTestWithStaticConfiguration; -import org.junit.*; +import org.junit.BeforeClass; +import org.junit.Before; +import org.junit.After; +import org.junit.Test; import com.google.common.collect.Lists; import com.google.common.io.Resources; - -@Ignore("Ignoring until SENTRY-1321 is complete") public class TestDbPrivilegeCleanupOnDrop extends AbstractTestWithStaticConfiguration { @@ -53,6 +55,8 @@ public class TestDbPrivilegeCleanupOnDrop extends private final static String tableName4 = "tb_4"; private final static String renameTag = "_new"; + static final long WAIT_FOR_NOTIFICATION_PROCESSING = 10000; + @BeforeClass public static void setupTestStaticConfiguration() throws Exception { useSentryService = true; @@ -72,6 +76,9 @@ public class TestDbPrivilegeCleanupOnDrop extends FileOutputStream to = new FileOutputStream(dataFile); Resources.copy(Resources.getResource(SINGLE_TYPE_DATA_FILE_NAME), to); to.close(); + while(!HMSFollower.isConnectedToHMS()) { + Thread.sleep(1000); + } } @After @@ -92,11 +99,12 @@ public class TestDbPrivilegeCleanupOnDrop extends public void testDropObjects() throws Exception { Connection connection = context.createConnection(ADMIN1); Statement statement = context.createStatement(connection); - setupRoles(statement); // create required roles setupDbObjects(statement); // create test DBs and Tables + Thread.sleep(5000);//TODO: Workaround for SENTRY-1422 setupPrivileges(statement); // setup privileges for USER1 dropDbObjects(statement); // drop objects + Thread.sleep(WAIT_FOR_NOTIFICATION_PROCESSING); verifyPrivilegesDropped(statement); // verify privileges are removed statement.close(); @@ -129,10 +137,12 @@ public class TestDbPrivilegeCleanupOnDrop extends Statement statement = context.createStatement(connection); setupRoles(statement); // create required roles setupDbObjects(statement); // create test DBs and Tables + Thread.sleep(5000);//TODO: Workaround for SENTRY-1422 setupPrivileges(statement); // setup privileges for USER1 dropDbObjects(statement); // drop DB and tables setupDbObjects(statement); // recreate same DBs and tables + Thread.sleep(WAIT_FOR_NOTIFICATION_PROCESSING); verifyPrivilegesDropped(statement); // verify the stale privileges removed } @@ -149,7 +159,9 @@ public class TestDbPrivilegeCleanupOnDrop extends setupRoles(statement); // create required roles setupDbObjects(statement); // create test DBs and Tables + Thread.sleep(5000);//TODO: Workaround for SENTRY-1422 setupPrivileges(statement); // setup privileges for USER1 + Thread.sleep(WAIT_FOR_NOTIFICATION_PROCESSING); // verify privileges on the created tables statement.execute("USE " + DB2); @@ -160,8 +172,8 @@ public class TestDbPrivilegeCleanupOnDrop extends tableName2); renameTables(statement); // alter tables to rename - // verify privileges removed for old tables + Thread.sleep(WAIT_FOR_NOTIFICATION_PROCESSING); verifyTablePrivilegesDropped(statement); // verify privileges created for new tables @@ -193,6 +205,7 @@ public class TestDbPrivilegeCleanupOnDrop extends statement.execute("USE " + DB1); statement.execute("CREATE TABLE t1 (c1 string)"); + Thread.sleep(5000);//TODO: Workaround for SENTRY-1422 // Grant SELECT/INSERT/DROP/ALTER to TABLE t1 statement.execute("GRANT SELECT ON TABLE t1 TO ROLE user_role"); statement.execute("GRANT INSERT ON TABLE t1 TO ROLE user_role"); @@ -206,10 +219,12 @@ public class TestDbPrivilegeCleanupOnDrop extends statement = context.createStatement(connection); statement.execute("USE " + DB1); statement.execute("ALTER TABLE t1 RENAME TO t2"); + Thread.sleep(WAIT_FOR_NOTIFICATION_PROCESSING); context.assertSentrySemanticException(statement, "drop table t1", semanticException); // After rename table t1 to t2, user_role should have permission to drop t2 statement.execute("drop table t2"); + Thread.sleep(WAIT_FOR_NOTIFICATION_PROCESSING); ResultSet resultSet = statement.executeQuery("SHOW GRANT ROLE user_role"); // user_role will revoke all privilege from table t2, only remain CREATE on db_1 assertRemainingRows(resultSet, 1); http://git-wip-us.apache.org/repos/asf/sentry/blob/2d5ed998/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/AbstractTestWithStaticConfiguration.java ---------------------------------------------------------------------- diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/AbstractTestWithStaticConfiguration.java b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/AbstractTestWithStaticConfiguration.java index 7dc3d0f..eae33e0 100644 --- a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/AbstractTestWithStaticConfiguration.java +++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/AbstractTestWithStaticConfiguration.java @@ -311,9 +311,6 @@ public abstract class AbstractTestWithStaticConfiguration { if ("true".equalsIgnoreCase(System.getProperty(ENABLE_SENTRY_HA, "false"))) { enableSentryHA = true; } - if (useSentryService && (!startSentry)) { - setupSentryService(); - } if (enableHiveConcurrency) { properties.put(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY.varname, "true"); @@ -322,9 +319,13 @@ public abstract class AbstractTestWithStaticConfiguration { properties.put(HiveConf.ConfVars.HIVE_LOCK_MANAGER.varname, "org.apache.hadoop.hive.ql.lockmgr.EmbeddedLockManager"); } + if (useSentryService && (!startSentry)) { + setupSentryService(); + } hiveServer = create(properties, baseDir, confDir, logDir, policyURI, fileSystem); hiveServer.start(); + createContext(); // Create tmp as scratch dir if it doesn't exist http://git-wip-us.apache.org/repos/asf/sentry/blob/2d5ed998/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/hiveserver/HiveServerFactory.java ---------------------------------------------------------------------- diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/hiveserver/HiveServerFactory.java b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/hiveserver/HiveServerFactory.java index 847da45..20db286 100644 --- a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/hiveserver/HiveServerFactory.java +++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/hiveserver/HiveServerFactory.java @@ -189,6 +189,10 @@ public class HiveServerFactory { properties.put(METASTORE_BYPASS, tempByPass); } + if (!properties.containsKey(HiveAuthzConf.AuthzConfVars.AUTHZ_SERVER_NAME.getVar())) { + properties.put(HiveAuthzConf.AuthzConfVars.AUTHZ_SERVER_NAME.getVar(), "server1"); + } + properties.put(METASTORE_SETUGI, "true"); properties.put(METASTORE_CLIENT_TIMEOUT, "100"); properties.put(ConfVars.HIVE_WAREHOUSE_SUBDIR_INHERIT_PERMS.varname, "true");
