Repository: sentry Updated Branches: refs/heads/sentry-ha-redesign 908072d66 -> 5a96dcdae (forced update)
http://git-wip-us.apache.org/repos/asf/sentry/blob/5a96dcda/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONAlterPartitionMessage.java ---------------------------------------------------------------------- diff --git a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONAlterPartitionMessage.java b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONAlterPartitionMessage.java deleted file mode 100644 index 99eb67a..0000000 --- a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONAlterPartitionMessage.java +++ /dev/null @@ -1,53 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.sentry.binding.metastore.messaging.json; - -import org.codehaus.jackson.annotate.JsonProperty; - -import java.util.List; - -public class SentryJSONAlterPartitionMessage extends JSONAlterPartitionMessage{ - @JsonProperty - String location; - @JsonProperty - String oldLocation; - - public SentryJSONAlterPartitionMessage() { - } - - public SentryJSONAlterPartitionMessage(String server, String servicePrincipal, String db, String table, - List<String> values, Long timestamp, String oldlocation, String newLocation) { - super(server, servicePrincipal, db, table, values, timestamp); - this.location = newLocation; - this.oldLocation = oldlocation; - } - - public String getLocation() { - return location; - } - - public String getOldLocation() { - return oldLocation; - } - - @Override - public String toString() { - return SentryJSONMessageDeserializer.serialize(this); - } -} http://git-wip-us.apache.org/repos/asf/sentry/blob/5a96dcda/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONAlterTableMessage.java ---------------------------------------------------------------------- diff --git a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONAlterTableMessage.java b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONAlterTableMessage.java deleted file mode 100644 index 6e59e25..0000000 --- a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONAlterTableMessage.java +++ /dev/null @@ -1,50 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.sentry.binding.metastore.messaging.json; - -import org.codehaus.jackson.annotate.JsonProperty; - -public class SentryJSONAlterTableMessage extends JSONAlterTableMessage { - @JsonProperty - String location; //newLocation - @JsonProperty - String oldLocation; - - public SentryJSONAlterTableMessage() { - } - - public SentryJSONAlterTableMessage(String server, String servicePrincipal, String db, String table, - Long timestamp, String oldLocation, String location) { - super(server, servicePrincipal, db, table, timestamp); - this.location = location; - this.oldLocation = oldLocation; - } - - public String getLocation() { - return location; - } - public String getOldLocation() { - return oldLocation; - } - - @Override - public String toString() { - return SentryJSONMessageDeserializer.serialize(this); - } -} http://git-wip-us.apache.org/repos/asf/sentry/blob/5a96dcda/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONCreateDatabaseMessage.java ---------------------------------------------------------------------- diff --git a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONCreateDatabaseMessage.java b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONCreateDatabaseMessage.java deleted file mode 100644 index ba19cbe..0000000 --- a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONCreateDatabaseMessage.java +++ /dev/null @@ -1,44 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.sentry.binding.metastore.messaging.json; - -import org.apache.hive.hcatalog.messaging.json.JSONCreateDatabaseMessage; -import org.codehaus.jackson.annotate.JsonProperty; - -public class SentryJSONCreateDatabaseMessage extends JSONCreateDatabaseMessage { - @JsonProperty - String location; - - public SentryJSONCreateDatabaseMessage() { - } - - public SentryJSONCreateDatabaseMessage(String server, String servicePrincipal, String db, Long timestamp, String location) { - super(server, servicePrincipal, db, timestamp); - this.location = location; - } - - public String getLocation() { - return location; - } - - @Override - public String toString() { - return SentryJSONMessageDeserializer.serialize(this); - } -} http://git-wip-us.apache.org/repos/asf/sentry/blob/5a96dcda/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONCreateTableMessage.java ---------------------------------------------------------------------- diff --git a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONCreateTableMessage.java b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONCreateTableMessage.java deleted file mode 100644 index 57d11d2..0000000 --- a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONCreateTableMessage.java +++ /dev/null @@ -1,45 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.sentry.binding.metastore.messaging.json; - -import org.apache.hive.hcatalog.messaging.json.JSONCreateTableMessage; -import org.codehaus.jackson.annotate.JsonProperty; - -public class SentryJSONCreateTableMessage extends JSONCreateTableMessage { - @JsonProperty - String location; - - public SentryJSONCreateTableMessage() { - } - - public SentryJSONCreateTableMessage(String server, String servicePrincipal, String db, String table, Long timestamp, String location) { - super(server, servicePrincipal, db, table, timestamp); - this.location = location; - } - - public String getLocation() { - return location; - } - - @Override - public String toString() { - return SentryJSONMessageDeserializer.serialize(this); - } - -} http://git-wip-us.apache.org/repos/asf/sentry/blob/5a96dcda/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONDropDatabaseMessage.java ---------------------------------------------------------------------- diff --git a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONDropDatabaseMessage.java b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONDropDatabaseMessage.java deleted file mode 100644 index 05f83f7..0000000 --- a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONDropDatabaseMessage.java +++ /dev/null @@ -1,44 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.sentry.binding.metastore.messaging.json; - -import org.apache.hive.hcatalog.messaging.json.JSONDropDatabaseMessage; -import org.codehaus.jackson.annotate.JsonProperty; - -public class SentryJSONDropDatabaseMessage extends JSONDropDatabaseMessage{ - @JsonProperty - String location; - - public SentryJSONDropDatabaseMessage() { - } - - public SentryJSONDropDatabaseMessage(String server, String servicePrincipal, String db, Long timestamp, String location) { - super(server, servicePrincipal, db, timestamp); - this.location = location; - } - - public String getLocation() { - return location; - } - - @Override - public String toString() { - return SentryJSONMessageDeserializer.serialize(this); - } -} http://git-wip-us.apache.org/repos/asf/sentry/blob/5a96dcda/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONDropPartitionMessage.java ---------------------------------------------------------------------- diff --git a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONDropPartitionMessage.java b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONDropPartitionMessage.java deleted file mode 100644 index 2ab61f7..0000000 --- a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONDropPartitionMessage.java +++ /dev/null @@ -1,49 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.sentry.binding.metastore.messaging.json; - -import org.apache.hive.hcatalog.messaging.json.JSONDropPartitionMessage; -import org.codehaus.jackson.annotate.JsonProperty; - -import java.util.List; -import java.util.Map; - -public class SentryJSONDropPartitionMessage extends JSONDropPartitionMessage { - @JsonProperty - String location; - - public SentryJSONDropPartitionMessage() { - } - - public SentryJSONDropPartitionMessage(String server, String servicePrincipal, String db, String table, - List<Map<String, String>> partitions, Long timestamp, String location) { - super(server, servicePrincipal, db, table, partitions, timestamp); - this.location = location; - } - - public String getLocation() { - return location; - } - - @Override - public String toString() { - return SentryJSONMessageDeserializer.serialize(this); - } - -} http://git-wip-us.apache.org/repos/asf/sentry/blob/5a96dcda/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONDropTableMessage.java ---------------------------------------------------------------------- diff --git a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONDropTableMessage.java b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONDropTableMessage.java deleted file mode 100644 index 7005776..0000000 --- a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONDropTableMessage.java +++ /dev/null @@ -1,45 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.sentry.binding.metastore.messaging.json; - -import org.apache.hive.hcatalog.messaging.json.JSONDropTableMessage; -import org.codehaus.jackson.annotate.JsonProperty; - - -public class SentryJSONDropTableMessage extends JSONDropTableMessage { - @JsonProperty - String location; - - public SentryJSONDropTableMessage() { - } - - public SentryJSONDropTableMessage(String server, String servicePrincipal, String db, String table, Long timestamp, String location) { - super(server, servicePrincipal, db, table, timestamp); - this.location = location; - } - - public String getLocation() { - return location; - } - - @Override - public String toString() { - return SentryJSONMessageDeserializer.serialize(this); - } -} http://git-wip-us.apache.org/repos/asf/sentry/blob/5a96dcda/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONMessageDeserializer.java ---------------------------------------------------------------------- diff --git a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONMessageDeserializer.java b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONMessageDeserializer.java deleted file mode 100644 index b645c45..0000000 --- a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONMessageDeserializer.java +++ /dev/null @@ -1,110 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.sentry.binding.metastore.messaging.json; - -import org.apache.hive.hcatalog.messaging.*; -import org.codehaus.jackson.map.DeserializationConfig; -import org.codehaus.jackson.map.ObjectMapper; - -public class SentryJSONMessageDeserializer extends MessageDeserializer { - static ObjectMapper mapper = new ObjectMapper(); - - public SentryJSONMessageDeserializer() { - } - - /** - * Method to de-serialize CreateDatabaseMessage instance. - */ - public SentryJSONCreateDatabaseMessage getCreateDatabaseMessage(String messageBody) { - try { - return (SentryJSONCreateDatabaseMessage)mapper.readValue(messageBody, SentryJSONCreateDatabaseMessage.class); - } catch (Exception var3) { - throw new IllegalArgumentException("Could not construct SentryJSONCreateDatabaseMessage.", var3); - } - } - - public SentryJSONDropDatabaseMessage getDropDatabaseMessage(String messageBody) { - try { - return (SentryJSONDropDatabaseMessage)mapper.readValue(messageBody, SentryJSONDropDatabaseMessage.class); - } catch (Exception var3) { - throw new IllegalArgumentException("Could not construct SentryJSONDropDatabaseMessage.", var3); - } - } - - public SentryJSONCreateTableMessage getCreateTableMessage(String messageBody) { - try { - return (SentryJSONCreateTableMessage)mapper.readValue(messageBody, SentryJSONCreateTableMessage.class); - } catch (Exception var3) { - throw new IllegalArgumentException("Could not construct SentryJSONCreateTableMessage.", var3); - } - } - - public SentryJSONAlterTableMessage getAlterTableMessage(String messageBody) { - try { - return (SentryJSONAlterTableMessage)mapper.readValue(messageBody, SentryJSONAlterTableMessage.class); - } catch (Exception var3) { - throw new IllegalArgumentException("Could not construct SentryJSONAlterTableMessage.", var3); - } - } - - public SentryJSONDropTableMessage getDropTableMessage(String messageBody) { - try { - return (SentryJSONDropTableMessage)mapper.readValue(messageBody, SentryJSONDropTableMessage.class); - } catch (Exception var3) { - throw new IllegalArgumentException("Could not construct SentryJSONDropTableMessage.", var3); - } - } - - public SentryJSONAddPartitionMessage getAddPartitionMessage(String messageBody) { - try { - return (SentryJSONAddPartitionMessage)mapper.readValue(messageBody, SentryJSONAddPartitionMessage.class); - } catch (Exception var3) { - throw new IllegalArgumentException("Could not construct SentryJSONAddPartitionMessage.", var3); - } - } - - public SentryJSONAlterPartitionMessage getAlterPartitionMessage(String messageBody) { - try { - return (SentryJSONAlterPartitionMessage)mapper.readValue(messageBody, SentryJSONAlterPartitionMessage.class); - } catch (Exception var3) { - throw new IllegalArgumentException("Could not construct SentryJSONAlterPartitionMessage.", var3); - } - } - - public SentryJSONDropPartitionMessage getDropPartitionMessage(String messageBody) { - try { - return (SentryJSONDropPartitionMessage)mapper.readValue(messageBody, SentryJSONDropPartitionMessage.class); - } catch (Exception var3) { - throw new IllegalArgumentException("Could not construct SentryJSONDropPartitionMessage.", var3); - } - } - - static { - mapper.configure(DeserializationConfig.Feature.FAIL_ON_UNKNOWN_PROPERTIES, false); - } - - public static String serialize(Object object) { - try { - return mapper.writeValueAsString(object); - } - catch (Exception exception) { - throw new IllegalArgumentException("Could not serialize: ", exception); - } - } -} http://git-wip-us.apache.org/repos/asf/sentry/blob/5a96dcda/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONMessageFactory.java ---------------------------------------------------------------------- diff --git a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONMessageFactory.java b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONMessageFactory.java deleted file mode 100644 index 00e7db8..0000000 --- a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONMessageFactory.java +++ /dev/null @@ -1,177 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.sentry.binding.metastore.messaging.json; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.hive.common.classification.InterfaceAudience; -import org.apache.hadoop.hive.common.classification.InterfaceStability; -import org.apache.hadoop.hive.metastore.api.Database; -import org.apache.hadoop.hive.metastore.api.FieldSchema; -import org.apache.hadoop.hive.metastore.api.Partition; -import org.apache.hadoop.hive.metastore.api.Table; -import org.apache.hadoop.hive.metastore.partition.spec.PartitionSpecProxy; -import org.apache.hive.hcatalog.messaging.*; - -import java.util.*; - -public class SentryJSONMessageFactory extends MessageFactory { - private static final Log LOG = LogFactory.getLog(SentryJSONMessageFactory.class.getName()); - private static SentryJSONMessageDeserializer deserializer = new SentryJSONMessageDeserializer(); - public SentryJSONMessageFactory() { - LOG.info("Using SentryJSONMessageFactory for building Notification log messages "); - - } - public MessageDeserializer getDeserializer() { - return deserializer; - } - - public String getVersion() { - return "0.1"; - } - - public String getMessageFormat() { - return "json"; - } - - public SentryJSONCreateDatabaseMessage buildCreateDatabaseMessage(Database db) { - return new SentryJSONCreateDatabaseMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, db.getName(), - Long.valueOf(this.now()), db.getLocationUri()); - } - public SentryJSONDropDatabaseMessage buildDropDatabaseMessage(Database db) { - return new SentryJSONDropDatabaseMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, db.getName(), - Long.valueOf(this.now()), db.getLocationUri()); - } - - public SentryJSONCreateTableMessage buildCreateTableMessage(Table table) { - return new SentryJSONCreateTableMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, table.getDbName(), - table.getTableName(), Long.valueOf(this.now()), table.getSd().getLocation()); - } - - public SentryJSONAlterTableMessage buildAlterTableMessage(Table before, Table after) { - return new SentryJSONAlterTableMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, before.getDbName(), - before.getTableName(), Long.valueOf(this.now()), before.getSd().getLocation(), after.getSd().getLocation()); - } - - public SentryJSONDropTableMessage buildDropTableMessage(Table table) { - return new SentryJSONDropTableMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, table.getDbName(), - table.getTableName(), Long.valueOf(this.now()), table.getSd().getLocation()); - } - - public SentryJSONAddPartitionMessage buildAddPartitionMessage(Table table, List<Partition> partitions) { - return new SentryJSONAddPartitionMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, table.getDbName(), - table.getTableName(), getPartitionKeyValues(table, partitions), Long.valueOf(this.now()), - getPartitionLocations(partitions)); - } - - private List<String> getPartitionLocations(List<Partition> partitions) { - List<String> paths = new ArrayList<String>(); - for(Partition partition:partitions) { - paths.add(partition.getSd().getLocation()); - } - return paths; - } - - //TODO: Not sure what is this used for. Need to investigate - private List<String> getPartitionLocations(PartitionSpecProxy partitionSpec) { - Iterator<Partition> iterator = partitionSpec.getPartitionIterator(); - List<String> locations = new ArrayList<String>(); - while(iterator.hasNext()) { - locations.add(iterator.next().getSd().getLocation()); - } - return locations; - } - - @InterfaceAudience.LimitedPrivate({"Hive"}) - @InterfaceStability.Evolving - public SentryJSONAddPartitionMessage buildAddPartitionMessage(Table table, PartitionSpecProxy partitionSpec) { - return new SentryJSONAddPartitionMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, table.getDbName(), - table.getTableName(), getPartitionKeyValues(table, partitionSpec), Long.valueOf(this.now()), - getPartitionLocations(partitionSpec)); - } - - public SentryJSONAlterPartitionMessage buildAlterPartitionMessage(Partition before, Partition after) { - /* - f (partitionEvent.getOldPartition() != null) { - oldLoc = partitionEvent.getOldPartition().getSd().getLocation(); - } - if (partitionEvent.getNewPartition() != null) { - newLoc = partitionEvent.getNewPartition().getSd().getLocation(); - } - - if ((oldLoc != null) && (newLoc != null) && (!oldLoc.equals(newLoc))) { - String authzObj = - partitionEvent.getOldPartition().getDbName() + "." - + partitionEvent.getOldPartition().getTableName(); - for (SentryMetastoreListenerPlugin plugin : sentryPlugins) { - plugin.renameAuthzObject(authzObj, oldLoc, - authzObj, newLoc); - } - } - * */ - return new SentryJSONAlterPartitionMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, before.getDbName(), - before.getTableName(), before.getValues(), Long.valueOf(this.now()), before.getSd().getLocation(), - after.getSd().getLocation()); - } - - public SentryJSONDropPartitionMessage buildDropPartitionMessage(Table table, Partition partition) { - return new SentryJSONDropPartitionMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, partition.getDbName(), - partition.getTableName(), Arrays.asList(getPartitionKeyValues(table, partition)), - Long.valueOf(this.now()), partition.getSd().getLocation()); - } - - private static Map<String, String> getPartitionKeyValues(Table table, Partition partition) { - LinkedHashMap partitionKeys = new LinkedHashMap(); - - for(int i = 0; i < table.getPartitionKeysSize(); ++i) { - partitionKeys.put(((FieldSchema)table.getPartitionKeys().get(i)).getName(), partition.getValues().get(i)); - } - - return partitionKeys; - } - - private static List<Map<String, String>> getPartitionKeyValues(Table table, List<Partition> partitions) { - ArrayList partitionList = new ArrayList(partitions.size()); - Iterator i$ = partitions.iterator(); - - while(i$.hasNext()) { - Partition partition = (Partition)i$.next(); - partitionList.add(getPartitionKeyValues(table, partition)); - } - - return partitionList; - } - - @InterfaceAudience.LimitedPrivate({"Hive"}) - @InterfaceStability.Evolving - private static List<Map<String, String>> getPartitionKeyValues(Table table, PartitionSpecProxy partitionSpec) { - ArrayList partitionList = new ArrayList(); - PartitionSpecProxy.PartitionIterator iterator = partitionSpec.getPartitionIterator(); - - while(iterator.hasNext()) { - Partition partition = (Partition)iterator.next(); - partitionList.add(getPartitionKeyValues(table, partition)); - } - - return partitionList; - } - //This is private in parent class - private long now() { - return System.currentTimeMillis() / 1000L; - } -} http://git-wip-us.apache.org/repos/asf/sentry/blob/5a96dcda/sentry-core/sentry-core-common/src/main/java/org/apache/sentry/core/common/exception/SentryInvalidHMSEventException.java ---------------------------------------------------------------------- diff --git a/sentry-core/sentry-core-common/src/main/java/org/apache/sentry/core/common/exception/SentryInvalidHMSEventException.java b/sentry-core/sentry-core-common/src/main/java/org/apache/sentry/core/common/exception/SentryInvalidHMSEventException.java new file mode 100644 index 0000000..50fb59c --- /dev/null +++ b/sentry-core/sentry-core-common/src/main/java/org/apache/sentry/core/common/exception/SentryInvalidHMSEventException.java @@ -0,0 +1,31 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sentry.core.common.exception; + +public class SentryInvalidHMSEventException extends SentryUserException { + private static final long serialVersionUID = 29620806553835L; + public SentryInvalidHMSEventException(String msg) { + super(msg); + } + public SentryInvalidHMSEventException(String msg, String reason) { + super(msg, reason); + } + public SentryInvalidHMSEventException(String msg, Throwable cause) { + super(msg, cause); + } +} http://git-wip-us.apache.org/repos/asf/sentry/blob/5a96dcda/sentry-core/sentry-core-common/src/main/java/org/apache/sentry/core/common/exception/SentryInvalidInputException.java ---------------------------------------------------------------------- diff --git a/sentry-core/sentry-core-common/src/main/java/org/apache/sentry/core/common/exception/SentryInvalidInputException.java b/sentry-core/sentry-core-common/src/main/java/org/apache/sentry/core/common/exception/SentryInvalidInputException.java index 903eddc..c36f166 100644 --- a/sentry-core/sentry-core-common/src/main/java/org/apache/sentry/core/common/exception/SentryInvalidInputException.java +++ b/sentry-core/sentry-core-common/src/main/java/org/apache/sentry/core/common/exception/SentryInvalidInputException.java @@ -25,4 +25,7 @@ public class SentryInvalidInputException extends SentryUserException { public SentryInvalidInputException(String msg, String reason) { super(msg, reason); } + public SentryInvalidInputException(String msg, Throwable cause) { + super(msg, cause); + } } http://git-wip-us.apache.org/repos/asf/sentry/blob/5a96dcda/sentry-provider/sentry-provider-db/pom.xml ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/pom.xml b/sentry-provider/sentry-provider-db/pom.xml index b8143ff..1699286 100644 --- a/sentry-provider/sentry-provider-db/pom.xml +++ b/sentry-provider/sentry-provider-db/pom.xml @@ -107,6 +107,16 @@ limitations under the License. <artifactId>sentry-policy-engine</artifactId> </dependency> <dependency> + <groupId>org.apache.sentry</groupId> + <artifactId>sentry-binding-hive-conf</artifactId> + <version>1.8.0-SNAPSHOT</version> + </dependency> + <dependency> + <groupId>org.apache.sentry</groupId> + <artifactId>sentry-binding-hive-follower</artifactId> + <version>1.8.0-SNAPSHOT</version> + </dependency> + <dependency> <groupId>org.apache.hive</groupId> <artifactId>hive-shims</artifactId> <scope>provided</scope> @@ -116,6 +126,10 @@ limitations under the License. <artifactId>hive-beeline</artifactId> </dependency> <dependency> + <groupId>org.apache.hive.hcatalog</groupId> + <artifactId>hive-hcatalog-server-extensions</artifactId> + </dependency> + <dependency> <groupId>org.apache.thrift</groupId> <artifactId>libfb303</artifactId> </dependency> @@ -177,6 +191,23 @@ limitations under the License. <groupId>org.apache.commons</groupId> <artifactId>commons-pool2</artifactId> </dependency> + <dependency> + <groupId>org.apache.hive</groupId> + <artifactId>hive-common</artifactId> + </dependency> + <dependency> + <groupId>org.apache.hive</groupId> + <artifactId>hive-metastore</artifactId> + </dependency> + <dependency> + <groupId>org.apache.hive</groupId> + <artifactId>hive-metastore</artifactId> + </dependency> + <dependency> + <groupId>org.apache.hive.hcatalog</groupId> + <artifactId>hive-hcatalog-server-extensions</artifactId> + <version>RELEASE</version> + </dependency> </dependencies> <build> http://git-wip-us.apache.org/repos/asf/sentry/blob/5a96dcda/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/HMSFollower.java ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/HMSFollower.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/HMSFollower.java new file mode 100644 index 0000000..4430471 --- /dev/null +++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/HMSFollower.java @@ -0,0 +1,411 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sentry.service.thrift; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.metastore.HiveMetaStoreClient; +import org.apache.hadoop.hive.metastore.api.MetaException; +import org.apache.hadoop.hive.metastore.api.NotificationEvent; +import org.apache.hadoop.hive.metastore.api.NotificationEventResponse; +import org.apache.hadoop.security.SaslRpcServer; +import org.apache.hive.hcatalog.messaging.HCatEventMessage; +import org.apache.sentry.binding.hive.conf.HiveAuthzConf; +import org.apache.sentry.core.common.exception.*; +import org.apache.sentry.provider.db.service.persistent.SentryStore; +import org.apache.sentry.provider.db.service.thrift.TSentryAuthorizable; +import org.apache.thrift.TException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.apache.sentry.binding.metastore.messaging.json.*; + +import javax.security.auth.Subject; +import javax.security.auth.login.LoginException; +import java.io.File; +import java.io.IOException; +import java.security.PrivilegedActionException; +import java.security.PrivilegedExceptionAction; +import java.util.List; + +import static org.apache.sentry.binding.hive.conf.HiveAuthzConf.AuthzConfVars.AUTHZ_SYNC_CREATE_WITH_POLICY_STORE; +import static org.apache.sentry.binding.hive.conf.HiveAuthzConf.AuthzConfVars.AUTHZ_SYNC_DROP_WITH_POLICY_STORE; + +/* +HMSFollower is the thread which follows the Hive MetaStore state changes from Sentry. +It gets the full update and notification logs from HMS and applies it to +update permissions stored in Sentry using SentryStore and also update the <obj,path> state +stored for HDFS- Sentry sync. + */ +@SuppressWarnings("PMD") +public class HMSFollower implements Runnable { + private static final Logger LOGGER = LoggerFactory.getLogger(HMSFollower.class); + + private long currentEventID; + private static boolean connectedToHMS = false; + private HiveMetaStoreClient client; + private SentryKerberosContext kerberosContext; + private Configuration authzConf; + private boolean kerberos; + private SentryStore sentryStore; + private String hiveInstance; + final static int maxRetriesForLogin = 3; + final static int maxRetriesForConnection = 3; + + HMSFollower(Configuration conf) throws SentryNoSuchObjectException, + SentryAccessDeniedException, SentrySiteConfigurationException, IOException { //TODO: Handle any possible exceptions or throw specific exceptions + LOGGER.info("HMSFollower is being initialized"); + authzConf = conf; + try { + sentryStore = new SentryStore(authzConf); + } catch (SentryStandbyException e) { + //TODO: Do not start HMSFollower if standby + } + //TODO: Initialize currentEventID from Sentry db + currentEventID = 0; + } + + @VisibleForTesting + HMSFollower(Configuration conf, SentryStore sentryStore, String hiveInstance) { + this.authzConf = conf; + this.sentryStore = sentryStore; + this.hiveInstance = hiveInstance; + } + + @VisibleForTesting + public static boolean isConnectedToHMS() { + return connectedToHMS; + } + + /* + Returns HMS Client if successful, returns null if HMS is not ready yet to take connections + Throws @LoginException if Kerberos context creation failed using Sentry's kerberos credentials + Throws @MetaException if there was a problem on creating an HMSClient + */ + private HiveMetaStoreClient getMetaStoreClient(Configuration conf) + throws LoginException, MetaException { + if(client != null) { + return client; + } + // Seems like HMS client creation although seems successful, + // it actually connects to an invalid HMS instance. + // So it seems like it is necessary to wait until we make sure metastore config is properly loaded. + boolean loadedHiveConf = HiveConf.isLoadMetastoreConfig(); + if(!loadedHiveConf) { + return null; + } + final HiveConf hiveConf = new HiveConf(); + hiveInstance = hiveConf.get(HiveAuthzConf.AuthzConfVars.AUTHZ_SERVER_NAME.getVar()); + + String principal, keytab; + + //TODO: Is this the right(standard) way to create a HMS client? HiveMetastoreClientFactoryImpl? + //TODO: Check if HMS is using kerberos instead of relying on Sentry conf + //TODO: Handle TGT renewals + kerberos = ServiceConstants.ServerConfig.SECURITY_MODE_KERBEROS.equalsIgnoreCase( + conf.get(ServiceConstants.ServerConfig.SECURITY_MODE, ServiceConstants.ServerConfig.SECURITY_MODE_KERBEROS).trim()); + if (kerberos) { + LOGGER.info("Making a kerberos connection to HMS"); + //TODO: Is this needed? Use Hadoop libraries to translate the _HOST placeholder with actual hostname + //Validate principal + principal = Preconditions.checkNotNull(ServiceConstants.ServerConfig.PRINCIPAL, + ServiceConstants.ServerConfig.PRINCIPAL + " is required"); + LOGGER.info("Using kerberos principal: " + principal); + final String[] principalParts = SaslRpcServer.splitKerberosName(principal); + Preconditions.checkArgument(principalParts.length == 3, + "Kerberos principal should have 3 parts: " + principal); + + keytab = Preconditions.checkNotNull(conf.get(ServiceConstants.ServerConfig.KEY_TAB), + ServiceConstants.ServerConfig.KEY_TAB + " is required"); + File keytabFile = new File(keytab); + Preconditions.checkState(keytabFile.isFile() && keytabFile.canRead(), + "Keytab " + keytab + " does not exist or is not readable."); + boolean establishedKerberosContext = false; + int attempt = 1; + while(establishedKerberosContext) { + try { + kerberosContext = new SentryKerberosContext(principal, keytab, true); + establishedKerberosContext = true; + LOGGER.info("Established kerberos context, will now connect to HMS"); + } catch (LoginException e) { + //Kerberos login failed + if( attempt > maxRetriesForLogin ) { + throw e; + } + attempt++; + } + } + boolean establishedConnection = false; + attempt = 1; + while(establishedConnection) { + try { + client = Subject.doAs(kerberosContext.getSubject(), new PrivilegedExceptionAction<HiveMetaStoreClient>() { + @Override + public HiveMetaStoreClient run() throws Exception { + return new HiveMetaStoreClient(hiveConf); + } + }); + LOGGER.info("Secure connection established with HMS"); + } catch (PrivilegedActionException e) { + if( attempt > maxRetriesForConnection ) { + //We should just retry as it is possible that HMS is not ready yet to receive requests + //TODO: How do we differentiate between kerberos problem versus HMS not being up? + LOGGER.error("Cannot connect to HMS", e); + } + attempt++; + } + } + } else { + //This is only for testing purposes. Sentry strongly recommends strong authentication + client = new HiveMetaStoreClient(hiveConf); + LOGGER.info("Non secure connection established with HMS"); + } + return client; + } + + public void run() { + if( client == null ) { + try { + client = getMetaStoreClient(authzConf); + if (client == null) { + //TODO: Do we want to throw an exception after a certain timeout? + return; + } else { + connectedToHMS = true; + LOGGER.info("HMSFollower of Sentry successfully connected to HMS"); + } + } catch (Exception e) { + + } + } + if (needFullUpdate()) { + //TODO: Handle + } else { + try { + NotificationEventResponse response = client.getNextNotification(currentEventID, Integer.MAX_VALUE, null); + if (response.isSetEvents()) { + LOGGER.info(String.format("CurrentEventID = %s. Processing %s events", + currentEventID, response.getEvents().size())); + processNotificationEvents(response.getEvents()); + } + } catch (TException e) { + LOGGER.error("ThriftException occured fetching Notification entries, will try"); + e.printStackTrace(); + } catch (SentryInvalidInputException|SentryInvalidHMSEventException e) { + throw new RuntimeException(e); + } + } + } + + private boolean needFullUpdate() { + //TODO Implement + return false; + } + + private boolean syncWithPolicyStore(HiveAuthzConf.AuthzConfVars syncConfVar) { + return "true" + .equalsIgnoreCase((authzConf.get(syncConfVar.getVar(), "true"))); + } + + /* + Throws SentryInvalidHMSEventException if Notification event contains insufficient information + */ + + void processNotificationEvents(List<NotificationEvent> events) throws + SentryInvalidHMSEventException, SentryInvalidInputException { + SentryJSONMessageDeserializer deserializer = new SentryJSONMessageDeserializer(); + + for (NotificationEvent event : events) { + String dbName, tableName, oldLocation, newLocation, location; + switch (HCatEventMessage.EventType.valueOf(event.getEventType())) { + case CREATE_DATABASE: + SentryJSONCreateDatabaseMessage message = deserializer.getCreateDatabaseMessage(event.getMessage()); + dbName = message.getDB(); + + location = message.getLocation(); + if (dbName == null || location == null) { + throw new SentryInvalidHMSEventException(String.format("Create database event has incomplete information. " + + "dbName = %s location = %s", dbName, location)); + } + if (syncWithPolicyStore(AUTHZ_SYNC_CREATE_WITH_POLICY_STORE)) { + try { + dropSentryDbPrivileges(dbName); + } catch (SentryNoSuchObjectException e) { + LOGGER.info("Drop Sentry privilege ignored as there are no privileges on the database: %s", dbName); + } catch (SentryInvalidInputException e) { + throw new SentryInvalidInputException("Could not process Create database event. Event: " + event.toString(), e); + } + } + //TODO: HDFSPlugin.addPath(dbName, location) + break; + case DROP_DATABASE: + SentryJSONDropDatabaseMessage dropDatabaseMessage = deserializer.getDropDatabaseMessage(event.getMessage()); + dbName = dropDatabaseMessage.getDB(); + if (dbName == null) { + throw new SentryInvalidHMSEventException(String.format("Drop database event has incomplete information. " + + "dbName = %s", dbName)); + } + if (syncWithPolicyStore(AUTHZ_SYNC_DROP_WITH_POLICY_STORE)) { + try { + dropSentryDbPrivileges(dbName); + } catch (SentryNoSuchObjectException e) { + LOGGER.info("Drop Sentry privilege ignored as there are no privileges on the database: %s", dbName); + } catch (SentryInvalidInputException e) { + throw new SentryInvalidInputException("Could not process Drop database event. Event: " + event.toString(), e); + } + } + //TODO: HDFSPlugin.deletePath(dbName, location) + break; + case CREATE_TABLE: + SentryJSONCreateTableMessage createTableMessage = deserializer.getCreateTableMessage(event.getMessage()); + dbName = createTableMessage.getDB(); + tableName = createTableMessage.getTable(); + location = createTableMessage.getLocation(); + if (dbName == null || tableName == null || location == null) { + throw new SentryInvalidHMSEventException(String.format("Create table event has incomplete information. " + + "dbName = %s, tableName = %s, location = %s", dbName, tableName, location)); + } + if (syncWithPolicyStore(AUTHZ_SYNC_CREATE_WITH_POLICY_STORE)) { + try { + dropSentryTablePrivileges(dbName, tableName); + } catch (SentryNoSuchObjectException e) { + LOGGER.info("Drop Sentry privilege ignored as there are no privileges on the table: %s.%s", dbName, tableName); + } catch (SentryInvalidInputException e) { + throw new SentryInvalidInputException("Could not process Create table event. Event: " + event.toString(), e); + } + } + //TODO: HDFSPlugin.deletePath(dbName, location) + break; + case DROP_TABLE: + SentryJSONDropTableMessage dropTableMessage = deserializer.getDropTableMessage(event.getMessage()); + dbName = dropTableMessage.getDB(); + tableName = dropTableMessage.getTable(); + if (dbName == null || tableName == null) { + throw new SentryInvalidHMSEventException(String.format("Drop table event has incomplete information. " + + "dbName = %s, tableName = %s", dbName, tableName)); + } + if (syncWithPolicyStore(AUTHZ_SYNC_DROP_WITH_POLICY_STORE)) { + try{ + dropSentryTablePrivileges(dbName, tableName); + } catch (SentryNoSuchObjectException e) { + LOGGER.info("Drop Sentry privilege ignored as there are no privileges on the table: %s.%s", dbName, tableName); + } catch (SentryInvalidInputException e) { + throw new SentryInvalidInputException("Could not process Drop table event. Event: " + event.toString(), e); + } + } + //TODO: HDFSPlugin.deletePath(dbName, location) + break; + case ALTER_TABLE: + SentryJSONAlterTableMessage alterTableMessage = deserializer.getAlterTableMessage(event.getMessage()); + + String oldDbName = alterTableMessage.getDB(); + String oldTableName = alterTableMessage.getTable(); + String newDbName = event.getDbName(); + String newTableName = event.getTableName(); + oldLocation = alterTableMessage.getOldLocation(); + newLocation = alterTableMessage.getLocation(); + + if (oldDbName == null || oldTableName == null || newDbName == null || newTableName == null || + oldLocation == null || newLocation == null) { + throw new SentryInvalidHMSEventException(String.format("Alter table event has incomplete information. " + + "oldDbName = %s, oldTableName = %s, oldLocation = %s, newDbName = %s, newTableName = %s, newLocation = %s", + oldDbName, oldTableName, oldLocation, newDbName, newTableName, newLocation)); + } + + if(!newDbName.equalsIgnoreCase(oldDbName) || !oldTableName.equalsIgnoreCase(newTableName)) { // Name has changed + if(!oldLocation.equals(newLocation)) { // Location has changed + + //Name and path has changed + // - Alter table rename for managed table + //TODO: Handle HDFS plugin + + } else { + //Only name has changed + // - Alter table rename for an external table + //TODO: Handle HDFS plugin + + } + try { + renamePrivileges(oldDbName, oldTableName, newDbName, newTableName); + } catch (SentryNoSuchObjectException e) { + LOGGER.info("Rename Sentry privilege ignored as there are no privileges on the table: %s.%s", oldDbName, oldTableName); + } catch (SentryInvalidInputException e) { + throw new SentryInvalidInputException("Could not process Alter table event. Event: " + event.toString(), e); + } catch (SentryStandbyException e) { + LOGGER.error("Seems like this process became a standby. " + + "Ignoring the failure. The new leader will reprocess this notification event"); + } + } else if(!oldLocation.equals(newLocation)) { // Only Location has changed{ + //- Alter table set location + //TODO: Handle HDFS plugin + } else { + LOGGER.info(String.format("Alter table notification ignored as neither name nor location has changed: " + + "oldDbName = %s, oldTableName = %s, oldLocation = %s, newDbName = %s, newTableName = %s, newLocation = %s", + oldDbName, oldTableName, oldLocation, newDbName, newTableName, newLocation)); + } + //TODO: Write test cases for all these cases + break; + case ADD_PARTITION: + case DROP_PARTITION: + case ALTER_PARTITION: + //TODO: Handle HDFS plugin + break; + } + currentEventID = event.getEventId(); + } + } + + private void dropSentryDbPrivileges(String dbName) throws SentryNoSuchObjectException, SentryInvalidInputException { + TSentryAuthorizable authorizable = new TSentryAuthorizable(hiveInstance); + authorizable.setDb(dbName); + try { + sentryStore.dropPrivilege(authorizable); + } catch (SentryStandbyException e) { + LOGGER.error("Seems like this process became a standby. " + + "Ignoring the failure. The new leader will reprocess this notification event"); + } + } + private void dropSentryTablePrivileges(String dbName, String tableName) throws SentryNoSuchObjectException, + SentryInvalidInputException { + TSentryAuthorizable authorizable = new TSentryAuthorizable(hiveInstance); + authorizable.setDb(dbName); + authorizable.setTable(tableName); + try { + sentryStore.dropPrivilege(authorizable); + } catch (SentryStandbyException e) { + LOGGER.error("Seems like this process became a standby. " + + "Ignoring the failure. The new leader will reprocess this notification event"); + } + } + private void renamePrivileges(String oldDbName, String oldTableName, String newDbName, String newTableName) throws + SentryNoSuchObjectException, SentryInvalidInputException, SentryStandbyException{ + TSentryAuthorizable oldAuthorizable = new TSentryAuthorizable(hiveInstance); + oldAuthorizable.setDb(oldDbName); + oldAuthorizable.setTable(oldTableName); + TSentryAuthorizable newAuthorizable = new TSentryAuthorizable(hiveInstance); + newAuthorizable.setDb(newDbName); + newAuthorizable.setTable(newTableName); + try { + sentryStore.renamePrivilege(oldAuthorizable, newAuthorizable); + } catch (SentryStandbyException e) { + LOGGER.error("Seems like this process became a standby. " + + "Ignoring the failure. The new leader will reprocess this notification event"); + } + } +} http://git-wip-us.apache.org/repos/asf/sentry/blob/5a96dcda/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryKerberosContext.java ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryKerberosContext.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryKerberosContext.java index f54f161..8d78d1d 100644 --- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryKerberosContext.java +++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryKerberosContext.java @@ -45,13 +45,17 @@ public class SentryKerberosContext implements Runnable { @Deprecated private boolean shutDownRenewer = false; - public SentryKerberosContext(String principal, String keyTab, boolean autoRenewTicket) + public SentryKerberosContext(String principal, String keyTab, boolean server) throws LoginException { subject = new Subject(false, Sets.newHashSet(new KerberosPrincipal(principal)), new HashSet<Object>(), new HashSet<Object>()); - kerberosConfig = KerberosConfiguration.createClientConfig(principal, new File(keyTab)); + if(server) { + kerberosConfig = KerberosConfiguration.createServerConfig(principal, new File(keyTab)); + } else { + kerberosConfig = KerberosConfiguration.createClientConfig(principal, new File(keyTab)); + } loginWithNewContext(); - if (autoRenewTicket) { + if (!server) { startRenewerThread(); } } http://git-wip-us.apache.org/repos/asf/sentry/blob/5a96dcda/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryService.java ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryService.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryService.java index e93e5b4..ddcb90c 100644 --- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryService.java +++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryService.java @@ -28,12 +28,7 @@ import java.security.PrivilegedExceptionAction; import java.util.ArrayList; import java.util.EventListener; import java.util.List; -import java.util.concurrent.Callable; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; -import java.util.concurrent.ThreadFactory; +import java.util.concurrent.*; import javax.security.auth.Subject; @@ -90,6 +85,7 @@ public class SentryService implements Callable { private final String[] principalParts; private final String keytab; private final ExecutorService serviceExecutor; + private ScheduledExecutorService hmsFollowerExecutor = null; private Future serviceStatus; private TServer thriftServer; private Status status; @@ -158,6 +154,14 @@ public class SentryService implements Callable { conf.set(SentryConstants.CURRENT_INCARNATION_ID_KEY, this.act.getIncarnationId()); webServerPort = conf.getInt(ServerConfig.SENTRY_WEB_PORT, ServerConfig.SENTRY_WEB_PORT_DEFAULT); + //TODO: Enable only if Hive is using Sentry? + try { + hmsFollowerExecutor = Executors.newScheduledThreadPool(1); + hmsFollowerExecutor.scheduleAtFixedRate(new HMSFollower(conf), 60000, 500, TimeUnit.MILLISECONDS); + }catch(Exception e) { + //TODO: Handle + LOGGER.error("Could not start HMSFollower"); + } status = Status.NOT_STARTED; } @@ -167,8 +171,7 @@ public class SentryService implements Callable { try { status = Status.STARTED; if (kerberos) { - Boolean autoRenewTicket = conf.getBoolean(ServerConfig.SENTRY_KERBEROS_TGT_AUTORENEW, ServerConfig.SENTRY_KERBEROS_TGT_AUTORENEW_DEFAULT); - kerberosContext = new SentryKerberosContext(principal, keytab, autoRenewTicket); + kerberosContext = new SentryKerberosContext(principal, keytab, true); Subject.doAs(kerberosContext.getSubject(), new PrivilegedExceptionAction<Void>() { @Override public Void run() throws Exception { @@ -319,6 +322,9 @@ public class SentryService implements Callable { } else { LOGGER.info("Sentry web service is already stopped..."); } + if(hmsFollowerExecutor != null) { + hmsFollowerExecutor.shutdown(); + } if (exception != null) { exception.ifExceptionThrow(); } http://git-wip-us.apache.org/repos/asf/sentry/blob/5a96dcda/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/service/thrift/TestHMSFollower.java ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/service/thrift/TestHMSFollower.java b/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/service/thrift/TestHMSFollower.java new file mode 100644 index 0000000..d601b1e --- /dev/null +++ b/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/service/thrift/TestHMSFollower.java @@ -0,0 +1,165 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sentry.service.thrift; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.metastore.HiveMetaStoreClient; +import org.apache.hadoop.hive.metastore.api.*; +import org.apache.hive.hcatalog.messaging.HCatEventMessage; +import org.apache.sentry.binding.metastore.messaging.json.SentryJSONMessageFactory; +import org.apache.sentry.provider.db.service.persistent.SentryStore; +import org.apache.sentry.provider.db.service.thrift.TSentryAuthorizable; +import org.junit.BeforeClass; +import org.junit.Test; +import org.mockito.Mockito; + +import java.util.ArrayList; +import java.util.List; + +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +public class TestHMSFollower { + SentryJSONMessageFactory messageFactory = new SentryJSONMessageFactory(); + SentryStore sentryStore = Mockito.mock(SentryStore.class); + final static String hiveInstance = "server2"; + + @Test + public void testCreateDatabase() throws Exception { + String dbName = "db1"; + + // Create notification events + NotificationEvent notificationEvent = new NotificationEvent(1, 0, HCatEventMessage.EventType.CREATE_DATABASE.toString(), + messageFactory.buildCreateDatabaseMessage(new Database(dbName, null, "hdfs://db1", null)).toString()); + List<NotificationEvent> events = new ArrayList<>(); + events.add(notificationEvent); + + Configuration configuration = new Configuration(); + HMSFollower hmsFollower = new HMSFollower(configuration, sentryStore, hiveInstance); + hmsFollower.processNotificationEvents(events); + + TSentryAuthorizable authorizable = new TSentryAuthorizable(hiveInstance); + authorizable.setServer(hiveInstance); + authorizable.setDb("db1"); + + verify(sentryStore, times(1)).dropPrivilege(authorizable); + } + @Test + public void testDropDatabase() throws Exception { + String dbName = "db1"; + + // Create notification events + NotificationEvent notificationEvent = new NotificationEvent(1, 0, HCatEventMessage.EventType.DROP_DATABASE.toString(), + messageFactory.buildDropDatabaseMessage(new Database(dbName, null, "hdfs://db1", null)).toString()); + List<NotificationEvent> events = new ArrayList<>(); + events.add(notificationEvent); + + Configuration configuration = new Configuration(); + HMSFollower hmsFollower = new HMSFollower(configuration, sentryStore, hiveInstance); + hmsFollower.processNotificationEvents(events); + + TSentryAuthorizable authorizable = new TSentryAuthorizable(hiveInstance); + authorizable.setServer(hiveInstance); + authorizable.setDb("db1"); + + verify(sentryStore, times(1)).dropPrivilege(authorizable); + } + @Test + public void testCreateTable() throws Exception { + String dbName = "db1"; + String tableName = "table1"; + + // Create notification events + StorageDescriptor sd = new StorageDescriptor(); + sd.setLocation("hdfs://db1.db/table1"); + NotificationEvent notificationEvent = new NotificationEvent(1, 0, HCatEventMessage.EventType.CREATE_TABLE.toString(), + messageFactory.buildCreateTableMessage(new Table(tableName, dbName, null, 0, 0, 0, sd, null, null, null, null, null)).toString()); + List<NotificationEvent> events = new ArrayList<>(); + events.add(notificationEvent); + + Configuration configuration = new Configuration(); + HMSFollower hmsFollower = new HMSFollower(configuration, sentryStore, hiveInstance); + hmsFollower.processNotificationEvents(events); + + TSentryAuthorizable authorizable = new TSentryAuthorizable(hiveInstance); + authorizable.setServer(hiveInstance); + authorizable.setDb("db1"); + authorizable.setTable(tableName); + + verify(sentryStore, times(1)).dropPrivilege(authorizable); + } + @Test + public void testDropTable() throws Exception { + String dbName = "db1"; + String tableName = "table1"; + + // Create notification events + StorageDescriptor sd = new StorageDescriptor(); + sd.setLocation("hdfs://db1.db/table1"); + NotificationEvent notificationEvent = new NotificationEvent(1, 0, HCatEventMessage.EventType.DROP_TABLE.toString(), + messageFactory.buildDropTableMessage(new Table(tableName, dbName, null, 0, 0, 0, sd, null, null, null, null, null)).toString()); + List<NotificationEvent> events = new ArrayList<>(); + events.add(notificationEvent); + + Configuration configuration = new Configuration(); + HMSFollower hmsFollower = new HMSFollower(configuration, sentryStore, hiveInstance); + hmsFollower.processNotificationEvents(events); + + TSentryAuthorizable authorizable = new TSentryAuthorizable(hiveInstance); + authorizable.setServer(hiveInstance); + authorizable.setDb("db1"); + authorizable.setTable(tableName); + + verify(sentryStore, times(1)).dropPrivilege(authorizable); + } + @Test + public void testRenameTable() throws Exception { + String dbName = "db1"; + String tableName = "table1"; + + String newDbName = "db1"; + String newTableName = "table2"; + + // Create notification events + StorageDescriptor sd = new StorageDescriptor(); + sd.setLocation("hdfs://db1.db/table1"); + NotificationEvent notificationEvent = new NotificationEvent(1, 0, HCatEventMessage.EventType.ALTER_TABLE.toString(), + messageFactory.buildAlterTableMessage( + new Table(tableName, dbName, null, 0, 0, 0, sd, null, null, null, null, null), + new Table(newTableName, newDbName, null, 0, 0, 0, sd, null, null, null, null, null)).toString()); + notificationEvent.setDbName(newDbName); + notificationEvent.setTableName(newTableName); + List<NotificationEvent> events = new ArrayList<>(); + events.add(notificationEvent); + + Configuration configuration = new Configuration(); + HMSFollower hmsFollower = new HMSFollower(configuration, sentryStore, hiveInstance); + hmsFollower.processNotificationEvents(events); + + TSentryAuthorizable authorizable = new TSentryAuthorizable(hiveInstance); + authorizable.setServer(hiveInstance); + authorizable.setDb(dbName); + authorizable.setTable(tableName); + + TSentryAuthorizable newAuthorizable = new TSentryAuthorizable(hiveInstance); + authorizable.setServer(hiveInstance); + newAuthorizable.setDb(newDbName); + newAuthorizable.setTable(newTableName); + + verify(sentryStore, times(1)).renamePrivilege(authorizable, newAuthorizable); + } +} http://git-wip-us.apache.org/repos/asf/sentry/blob/5a96dcda/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/dbprovider/TestDbPrivilegeCleanupOnDrop.java ---------------------------------------------------------------------- diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/dbprovider/TestDbPrivilegeCleanupOnDrop.java b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/dbprovider/TestDbPrivilegeCleanupOnDrop.java index 439b9de..69fc8d2 100644 --- a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/dbprovider/TestDbPrivilegeCleanupOnDrop.java +++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/dbprovider/TestDbPrivilegeCleanupOnDrop.java @@ -31,14 +31,16 @@ import java.sql.Statement; import java.util.ArrayList; import java.util.List; +import org.apache.sentry.service.thrift.HMSFollower; import org.apache.sentry.tests.e2e.hive.AbstractTestWithStaticConfiguration; -import org.junit.*; +import org.junit.BeforeClass; +import org.junit.Before; +import org.junit.After; +import org.junit.Test; import com.google.common.collect.Lists; import com.google.common.io.Resources; - -@Ignore("Ignoring until SENTRY-1321 is complete") public class TestDbPrivilegeCleanupOnDrop extends AbstractTestWithStaticConfiguration { @@ -53,6 +55,8 @@ public class TestDbPrivilegeCleanupOnDrop extends private final static String tableName4 = "tb_4"; private final static String renameTag = "_new"; + static final long WAIT_FOR_NOTIFICATION_PROCESSING = 10000; + @BeforeClass public static void setupTestStaticConfiguration() throws Exception { useSentryService = true; @@ -72,6 +76,9 @@ public class TestDbPrivilegeCleanupOnDrop extends FileOutputStream to = new FileOutputStream(dataFile); Resources.copy(Resources.getResource(SINGLE_TYPE_DATA_FILE_NAME), to); to.close(); + while(!HMSFollower.isConnectedToHMS()) { + Thread.sleep(1000); + } } @After @@ -92,11 +99,12 @@ public class TestDbPrivilegeCleanupOnDrop extends public void testDropObjects() throws Exception { Connection connection = context.createConnection(ADMIN1); Statement statement = context.createStatement(connection); - setupRoles(statement); // create required roles setupDbObjects(statement); // create test DBs and Tables + Thread.sleep(5000);//TODO: Workaround for SENTRY-1422 setupPrivileges(statement); // setup privileges for USER1 dropDbObjects(statement); // drop objects + Thread.sleep(WAIT_FOR_NOTIFICATION_PROCESSING); verifyPrivilegesDropped(statement); // verify privileges are removed statement.close(); @@ -129,10 +137,12 @@ public class TestDbPrivilegeCleanupOnDrop extends Statement statement = context.createStatement(connection); setupRoles(statement); // create required roles setupDbObjects(statement); // create test DBs and Tables + Thread.sleep(5000);//TODO: Workaround for SENTRY-1422 setupPrivileges(statement); // setup privileges for USER1 dropDbObjects(statement); // drop DB and tables setupDbObjects(statement); // recreate same DBs and tables + Thread.sleep(WAIT_FOR_NOTIFICATION_PROCESSING); verifyPrivilegesDropped(statement); // verify the stale privileges removed } @@ -149,7 +159,9 @@ public class TestDbPrivilegeCleanupOnDrop extends setupRoles(statement); // create required roles setupDbObjects(statement); // create test DBs and Tables + Thread.sleep(5000);//TODO: Workaround for SENTRY-1422 setupPrivileges(statement); // setup privileges for USER1 + Thread.sleep(WAIT_FOR_NOTIFICATION_PROCESSING); // verify privileges on the created tables statement.execute("USE " + DB2); @@ -160,8 +172,8 @@ public class TestDbPrivilegeCleanupOnDrop extends tableName2); renameTables(statement); // alter tables to rename - // verify privileges removed for old tables + Thread.sleep(WAIT_FOR_NOTIFICATION_PROCESSING); verifyTablePrivilegesDropped(statement); // verify privileges created for new tables @@ -193,6 +205,7 @@ public class TestDbPrivilegeCleanupOnDrop extends statement.execute("USE " + DB1); statement.execute("CREATE TABLE t1 (c1 string)"); + Thread.sleep(5000);//TODO: Workaround for SENTRY-1422 // Grant SELECT/INSERT/DROP/ALTER to TABLE t1 statement.execute("GRANT SELECT ON TABLE t1 TO ROLE user_role"); statement.execute("GRANT INSERT ON TABLE t1 TO ROLE user_role"); @@ -206,10 +219,12 @@ public class TestDbPrivilegeCleanupOnDrop extends statement = context.createStatement(connection); statement.execute("USE " + DB1); statement.execute("ALTER TABLE t1 RENAME TO t2"); + Thread.sleep(WAIT_FOR_NOTIFICATION_PROCESSING); context.assertSentrySemanticException(statement, "drop table t1", semanticException); // After rename table t1 to t2, user_role should have permission to drop t2 statement.execute("drop table t2"); + Thread.sleep(WAIT_FOR_NOTIFICATION_PROCESSING); ResultSet resultSet = statement.executeQuery("SHOW GRANT ROLE user_role"); // user_role will revoke all privilege from table t2, only remain CREATE on db_1 assertRemainingRows(resultSet, 1); http://git-wip-us.apache.org/repos/asf/sentry/blob/5a96dcda/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/AbstractTestWithStaticConfiguration.java ---------------------------------------------------------------------- diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/AbstractTestWithStaticConfiguration.java b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/AbstractTestWithStaticConfiguration.java index 7dc3d0f..eae33e0 100644 --- a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/AbstractTestWithStaticConfiguration.java +++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/AbstractTestWithStaticConfiguration.java @@ -311,9 +311,6 @@ public abstract class AbstractTestWithStaticConfiguration { if ("true".equalsIgnoreCase(System.getProperty(ENABLE_SENTRY_HA, "false"))) { enableSentryHA = true; } - if (useSentryService && (!startSentry)) { - setupSentryService(); - } if (enableHiveConcurrency) { properties.put(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY.varname, "true"); @@ -322,9 +319,13 @@ public abstract class AbstractTestWithStaticConfiguration { properties.put(HiveConf.ConfVars.HIVE_LOCK_MANAGER.varname, "org.apache.hadoop.hive.ql.lockmgr.EmbeddedLockManager"); } + if (useSentryService && (!startSentry)) { + setupSentryService(); + } hiveServer = create(properties, baseDir, confDir, logDir, policyURI, fileSystem); hiveServer.start(); + createContext(); // Create tmp as scratch dir if it doesn't exist http://git-wip-us.apache.org/repos/asf/sentry/blob/5a96dcda/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/hiveserver/HiveServerFactory.java ---------------------------------------------------------------------- diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/hiveserver/HiveServerFactory.java b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/hiveserver/HiveServerFactory.java index 847da45..20db286 100644 --- a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/hiveserver/HiveServerFactory.java +++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/hiveserver/HiveServerFactory.java @@ -189,6 +189,10 @@ public class HiveServerFactory { properties.put(METASTORE_BYPASS, tempByPass); } + if (!properties.containsKey(HiveAuthzConf.AuthzConfVars.AUTHZ_SERVER_NAME.getVar())) { + properties.put(HiveAuthzConf.AuthzConfVars.AUTHZ_SERVER_NAME.getVar(), "server1"); + } + properties.put(METASTORE_SETUGI, "true"); properties.put(METASTORE_CLIENT_TIMEOUT, "100"); properties.put(ConfVars.HIVE_WAREHOUSE_SUBDIR_INHERIT_PERMS.varname, "true");
