Repository: incubator-atlas Updated Branches: refs/heads/master 5ab199511 -> 868cd831b
ATLAS-871 Make audit repository implementation configurable (jnhagelb via shwethags) Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/868cd831 Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/868cd831 Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/868cd831 Branch: refs/heads/master Commit: 868cd831b8add7a2353a1447aaf41666e402b677 Parents: 5ab1995 Author: Shwetha GS <[email protected]> Authored: Fri Jun 10 14:20:39 2016 +0530 Committer: Shwetha GS <[email protected]> Committed: Fri Jun 10 14:20:39 2016 +0530 ---------------------------------------------------------------------- distro/pom.xml | 2 + distro/src/bin/atlas_start.py | 26 +++++----- distro/src/conf/atlas-application.properties | 21 ++++++++ pom.xml | 2 + release-log.txt | 1 + .../apache/atlas/RepositoryMetadataModule.java | 32 ++++++++++--- .../audit/InMemoryEntityAuditRepository.java | 3 ++ .../audit/NoopEntityAuditRepository.java | 50 ++++++++++++++++++++ .../main/resources/atlas-application.properties | 3 ++ 9 files changed, 122 insertions(+), 18 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/868cd831/distro/pom.xml ---------------------------------------------------------------------- diff --git a/distro/pom.xml b/distro/pom.xml index 131a488..7aa4e50 100644 --- a/distro/pom.xml +++ b/distro/pom.xml @@ -52,6 +52,7 @@ atlas.graph.index.search.solr.zookeeper-url= </titan.index.properties> <hbase.embedded>false</hbase.embedded> <solr.embedded>false</solr.embedded> + <entity.repository.properties>#atlas.EntityAuditRepository.impl=org.apache.atlas.repository.audit.HBaseBasedAuditRepository</entity.repository.properties> </properties> <profiles> @@ -130,6 +131,7 @@ atlas.graph.index.search.elasticsearch.client-only=false atlas.graph.index.search.elasticsearch.local-mode=true atlas.graph.index.search.elasticsearch.create.sleep=2000 </titan.index.properties> + <entity.repository.properties>atlas.EntityAuditRepository.impl=org.apache.atlas.repository.audit.NoopEntityAuditRepository</entity.repository.properties> </properties> </profile> http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/868cd831/distro/src/bin/atlas_start.py ---------------------------------------------------------------------- diff --git a/distro/src/bin/atlas_start.py b/distro/src/bin/atlas_start.py index 1ba38f8..bb61898 100755 --- a/distro/src/bin/atlas_start.py +++ b/distro/src/bin/atlas_start.py @@ -72,32 +72,36 @@ def main(): web_app_dir = mc.webAppDir(atlas_home) mc.expandWebApp(atlas_home) - #add hbase-site.xml to classpath - hbase_conf_dir = mc.hbaseConfDir(atlas_home) - p = os.pathsep atlas_classpath = confdir + p \ + os.path.join(web_app_dir, "atlas", "WEB-INF", "classes" ) + p \ + os.path.join(web_app_dir, "atlas", "WEB-INF", "lib", "atlas-titan-${project.version}.jar" ) + p \ + os.path.join(web_app_dir, "atlas", "WEB-INF", "lib", "*" ) + p \ + os.path.join(atlas_home, "libext", "*") - if os.path.exists(hbase_conf_dir): - atlas_classpath = atlas_classpath + p \ + + is_hbase = mc.is_hbase(confdir) + + if is_hbase: + #add hbase-site.xml to classpath + hbase_conf_dir = mc.hbaseConfDir(atlas_home) + + if os.path.exists(hbase_conf_dir): + atlas_classpath = atlas_classpath + p \ + hbase_conf_dir - else: - if mc.is_hbase(confdir): - raise Exception("Could not find hbase-site.xml in %s. Please set env var HBASE_CONF_DIR to the hbase client conf dir", hbase_conf_dir) + else: + if mc.is_hbase(confdir): + raise Exception("Could not find hbase-site.xml in %s. Please set env var HBASE_CONF_DIR to the hbase client conf dir", hbase_conf_dir) if mc.isCygwin(): atlas_classpath = mc.convertCygwinPath(atlas_classpath, True) atlas_pid_file = mc.pidFile(atlas_home) - + if os.path.isfile(atlas_pid_file): #Check if process listed in atlas.pid file is still running pf = file(atlas_pid_file, 'r') pid = pf.read().strip() - pf.close() + pf.close() if mc.exist_pid((int)(pid)): if is_setup: @@ -106,7 +110,7 @@ def main(): else: mc.server_pid_not_running(pid) - if mc.is_hbase_local(confdir): + if is_hbase and mc.is_hbase_local(confdir): print "configured for local hbase." mc.configure_hbase(atlas_home) mc.run_hbase_action(mc.hbaseBinDir(atlas_home), "start", hbase_conf_dir, logdir) http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/868cd831/distro/src/conf/atlas-application.properties ---------------------------------------------------------------------- diff --git a/distro/src/conf/atlas-application.properties b/distro/src/conf/atlas-application.properties index 873c1e0..bdd02d6 100755 --- a/distro/src/conf/atlas-application.properties +++ b/distro/src/conf/atlas-application.properties @@ -22,6 +22,27 @@ atlas.graph.storage.backend=${titan.storage.backend} ${titan.storage.properties} +# Delete handler +# +# This allows the default behavior of doing "soft" deletes to be changed. +# +# Allowed Values: +# org.apache.atlas.repository.graph.SoftDeleteHandler - all deletes are "soft" deletes +# org.apache.atlas.repository.graph.HardDeleteHandler - all deletes are "hard" deletes +# +#atlas.DeleteHandler.impl=org.apache.atlas.repository.graph.SoftDeleteHandler + +# Entity audit repository +# +# This allows the default behavior of logging entity changes to hbase to be changed. +# +# Allowed Values: +# org.apache.atlas.repository.audit.HBaseBasedAuditRepository - log entity changes to hbase +# org.apache.atlas.repository.audit.NoopEntityAuditRepository - disable the audit repository +# +${entity.repository.properties} + + # Graph Search Index atlas.graph.index.search.backend=${titan.index.backend} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/868cd831/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index eac95c8..e13345e 100755 --- a/pom.xml +++ b/pom.xml @@ -411,6 +411,7 @@ <titan.storage.backend>berkeleyje</titan.storage.backend> <titan.index.backend>elasticsearch</titan.index.backend> + <entity.repository.impl>org.apache.atlas.repository.audit.InMemoryEntityAuditRepository</entity.repository.impl> </properties> <profiles> @@ -425,6 +426,7 @@ <titan.index.backend>solr5</titan.index.backend> <solr.zk.address>localhost:9983</solr.zk.address> <titan.storage.hostname>localhost</titan.storage.hostname> + <entity.repository.impl>org.apache.atlas.repository.audit.HBaseBasedAuditRepository</entity.repository.impl> </properties> </profile> http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/868cd831/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index a43ac40..afbd09b 100644 --- a/release-log.txt +++ b/release-log.txt @@ -22,6 +22,7 @@ ATLAS-409 Atlas will not import avro tables with schema read from a file (dosset ATLAS-379 Create sqoop and falcon metadata addons (venkatnrangan,bvellanki,sowmyaramesh via shwethags) ALL CHANGES: +ATLAS-871 Make audit repository implementation configurable (jnhagelb via shwethags) ATLAS-885 optimize HBaseStoreManager to avoid expensive HTable instantiation every 5 seconds (madhan.neethiraj via yhemanth) ATLAS-878 UI: Not showing details of SD, DB and COLUMNS (saqeeb.s via shwethags) ATLAS-853 User's name to be mentioned in the top user drop down (saqeeb.s via shwethags) http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/868cd831/repository/src/main/java/org/apache/atlas/RepositoryMetadataModule.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/RepositoryMetadataModule.java b/repository/src/main/java/org/apache/atlas/RepositoryMetadataModule.java index 15201dd..c4c7678 100755 --- a/repository/src/main/java/org/apache/atlas/RepositoryMetadataModule.java +++ b/repository/src/main/java/org/apache/atlas/RepositoryMetadataModule.java @@ -87,7 +87,7 @@ public class RepositoryMetadataModule extends com.google.inject.AbstractModule { bindAuditRepository(binder()); - bind(DeleteHandler.class).to(getDeleteHandler()).asEagerSingleton(); + bind(DeleteHandler.class).to(getDeleteHandlerImpl()).asEagerSingleton(); //Add EntityAuditListener as EntityChangeListener Multibinder<EntityChangeListener> entityChangeListenerBinder = @@ -100,17 +100,35 @@ public class RepositoryMetadataModule extends com.google.inject.AbstractModule { } protected void bindAuditRepository(Binder binder) { - //Map EntityAuditRepository interface to hbase based implementation - binder.bind(EntityAuditRepository.class).to(HBaseBasedAuditRepository.class).asEagerSingleton(); - //Add HBaseBasedAuditRepository to service so that connection is closed at shutdown - Multibinder<Service> serviceBinder = Multibinder.newSetBinder(binder, Service.class); - serviceBinder.addBinding().to(HBaseBasedAuditRepository.class); + Class<? extends EntityAuditRepository> auditRepoImpl = getAuditRepositoryImpl(); + + //Map EntityAuditRepository interface to configured implementation + binder.bind(EntityAuditRepository.class).to(auditRepoImpl).asEagerSingleton(); + + if(Service.class.isAssignableFrom(auditRepoImpl)) { + Class<? extends Service> auditRepoService = (Class<? extends Service>)auditRepoImpl; + //if it's a service, make sure that it gets properly closed at shutdown + Multibinder<Service> serviceBinder = Multibinder.newSetBinder(binder, Service.class); + serviceBinder.addBinding().to(auditRepoService); + } + } + + + private static final String AUDIT_REPOSITORY_IMPLEMENTATION_PROPERTY = "atlas.EntityAuditRepository.impl"; + + private Class<? extends EntityAuditRepository> getAuditRepositoryImpl() { + try { + return ApplicationProperties.getClass(AUDIT_REPOSITORY_IMPLEMENTATION_PROPERTY, + HBaseBasedAuditRepository.class.getName(), EntityAuditRepository.class); + } catch (AtlasException e) { + throw new RuntimeException(e); + } } private static final String DELETE_HANDLER_IMPLEMENTATION_PROPERTY = "atlas.DeleteHandler.impl"; - private Class<? extends DeleteHandler> getDeleteHandler() { + private Class<? extends DeleteHandler> getDeleteHandlerImpl() { try { return ApplicationProperties.getClass(DELETE_HANDLER_IMPLEMENTATION_PROPERTY, SoftDeleteHandler.class.getName(), DeleteHandler.class); http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/868cd831/repository/src/main/java/org/apache/atlas/repository/audit/InMemoryEntityAuditRepository.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/audit/InMemoryEntityAuditRepository.java b/repository/src/main/java/org/apache/atlas/repository/audit/InMemoryEntityAuditRepository.java index 93f224f..cf76596 100644 --- a/repository/src/main/java/org/apache/atlas/repository/audit/InMemoryEntityAuditRepository.java +++ b/repository/src/main/java/org/apache/atlas/repository/audit/InMemoryEntityAuditRepository.java @@ -21,6 +21,8 @@ package org.apache.atlas.repository.audit; import org.apache.atlas.AtlasException; import org.apache.atlas.EntityAuditEvent; +import com.google.inject.Singleton; + import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -30,6 +32,7 @@ import java.util.TreeMap; /** * Entity audit repository where audit events are stored in-memory. Used only for integration tests */ +@Singleton public class InMemoryEntityAuditRepository implements EntityAuditRepository { private TreeMap<String, EntityAuditEvent> auditEvents = new TreeMap<>(); http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/868cd831/repository/src/main/java/org/apache/atlas/repository/audit/NoopEntityAuditRepository.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/audit/NoopEntityAuditRepository.java b/repository/src/main/java/org/apache/atlas/repository/audit/NoopEntityAuditRepository.java new file mode 100644 index 0000000..9f77bfe --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/repository/audit/NoopEntityAuditRepository.java @@ -0,0 +1,50 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p/> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p/> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.atlas.repository.audit; + +import java.util.Collections; +import java.util.List; + +import org.apache.atlas.AtlasException; +import org.apache.atlas.EntityAuditEvent; + +import com.google.inject.Singleton; + +/** + * Implementation that completely disables the audit repository. + */ +@Singleton +public class NoopEntityAuditRepository implements EntityAuditRepository { + + @Override + public void putEvents(EntityAuditEvent... events) throws AtlasException { + //do nothing + } + + @Override + public synchronized void putEvents(List<EntityAuditEvent> events) throws AtlasException { + //do nothing + } + + @Override + public List<EntityAuditEvent> listEvents(String entityId, String startKey, short maxResults) + throws AtlasException { + return Collections.emptyList(); + } +} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/868cd831/typesystem/src/main/resources/atlas-application.properties ---------------------------------------------------------------------- diff --git a/typesystem/src/main/resources/atlas-application.properties b/typesystem/src/main/resources/atlas-application.properties index 3318ff3..11253e6 100644 --- a/typesystem/src/main/resources/atlas-application.properties +++ b/typesystem/src/main/resources/atlas-application.properties @@ -23,6 +23,9 @@ atlas.rest.address=http://localhost:31000 # Graph Storage atlas.graph.storage.backend=${titan.storage.backend} +# Entity repository implementation +atlas.EntityAuditRepository.impl=${entity.repository.impl} + # Graph Search Index Backend atlas.graph.index.search.backend=${titan.index.backend}
