Repository: incubator-atlas Updated Branches: refs/heads/master 141b78c16 -> b63d6316e
ATLAS-1535 : Some webapp tests are failing due to a stale Titan transaction Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/b63d6316 Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/b63d6316 Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/b63d6316 Branch: refs/heads/master Commit: b63d6316e3b3b9ad7bd96d49914ffc724c0b80fa Parents: 141b78c Author: Jeff Hagelberg <[email protected]> Authored: Thu Feb 9 18:17:30 2017 -0500 Committer: Jeff Hagelberg <[email protected]> Committed: Thu Feb 9 18:17:30 2017 -0500 ---------------------------------------------------------------------- release-log.txt | 33 +++++----- .../filters/StaleTransactionCleanupFilter.java | 63 ++++++++++++++++++++ .../atlas/web/listeners/GuiceServletConfig.java | 2 + .../apache/atlas/web/resources/BaseService.java | 11 ---- .../web/resources/TaxonomyServiceTest.java | 40 ++++++++----- 5 files changed, 108 insertions(+), 41 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/b63d6316/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index 9cb1bda..d544833 100644 --- a/release-log.txt +++ b/release-log.txt @@ -9,13 +9,14 @@ ATLAS-1060 Add composite indexes for exact match performance improvements for al ATLAS-1127 Modify creation and modification timestamps to Date instead of Long(sumasai) ALL CHANGES: +ATLAS-1535 Some webapp tests are failing due to a stale Titan transaction (jnhagelberg) ATLAS-1401 Document in detail how to set up Eclipse for Atlas dev environment -ATLAS-1527 Batch entity retrievals - DefaultMetadataService.loadEntities -ATLAS-1385 Add configuration property to disable full text mapper (wwojcik via jnhagelb) -ATLAS-746 After updating a set of entities, response contains only the first entity definition (jnhagelb) -ATLAS-1510 Consolidate/batch calls to GraphBackedTypeStore.findVertex() (jnhagelb) -ATLAS-1388 Cache entities that are created/updated (jnhagelb) -ATLAS-1369 Optimize Gremlin queries generated by DSL translator (jnhagelb) +ATLAS-1527 Batch entity retrievals - DefaultMetadataService.loadEntities (wojciechwojcik via jnhagelberg) +ATLAS-1385 Add configuration property to disable full text mapper (wojciechwojcik via jnhagelberg) +ATLAS-746 After updating a set of entities, response contains only the first entity definition (jnhagelberg) +ATLAS-1510 Consolidate/batch calls to GraphBackedTypeStore.findVertex() (jnhagelberg) +ATLAS-1388 Cache entities that are created/updated (jnhagelberg) +ATLAS-1369 Optimize Gremlin queries generated by DSL translator (jnhagelberg) ATLAS-1517: updated hive_model to include schema related attributes ([email protected] via mneethiraj) ATLAS-1514 Remove duplicates from class array attribute when target is deleted (dkantor) ATLAS-1509: fixed issues with deletion during updates (sumasai via mneethiraj) @@ -29,7 +30,7 @@ ATLAS-1431 Integration tests added by ATLAS 1311 fail on Eclipse (david_radley v ATLAS-1387 Compiled Query Cache ([email protected] via svimal2106) ATLAS-1312 Update QuickStart to use the v2 APIs for types and entities creation ([email protected] via mneethiraj) ATLAS-1498 added unit-tests to validate handling of array/map/struct attributes in entity create/update (sumasai via mneethiraj) -ATLAS-1114 Performance improvements for create/update entity (jnhagelb) +ATLAS-1114 Performance improvements for create/update entity (jnhagelberg) ATLAS-1403 Perf and stability improvements to DSL search and lineage query execution ([email protected] via mneethiraj) ATLAS-1425 Integrate Discovery/Search API in Atlas UI (kevalbhatt via mneethiraj) ATLAS-1482 UI update in assigning a tag to multiple entities using single API call (kevalbhatt via mneethiraj) @@ -42,7 +43,7 @@ ATLAS-1437 UI update to disallow tag association changes to deleted entities (Ka ATLAS-1352 fix for error in redirecting to Knox gateway URL (nixonrodrigues via mneethiraj) ATLAS-1467 instance create/full-Update implementation (sumasai via mneethiraj) ATLAS-1463 option to exclude specific entity attributes in audit records ([email protected] via mneethiraj) -ATLAS-1386 Avoid uunnecessary type cache lookups (jnhagelb) +ATLAS-1386 Avoid uunnecessary type cache lookups (jnhagelberg) ATLAS-1000 added build instructions to README.txt (mneethiraj) ATLAS-1471 avoid unnecessary overhead in debug log calls (mneethiraj) ATLAS-1464 option to include only specified attributes in notification message ([email protected] via mneethiraj) @@ -145,7 +146,7 @@ ATLAS-1233 UnitTests for the TypeDefStores (apoorvnaik via sumasai) ATLAS-1240 Adding Change listeners to react on changes in TypesDef (apoorvnaik via sumasai) ATLAS-1239 when stopping Atlas on the command line it should explicitly say when it has stopped (ayubkhan via sumasai) ATLAS-1253 Extract error codes into AtlasErrorCode Enum (apoorvnaik via sumasai) -ATLAS-1195 Clean up DSL Translation (jnhagelb via dkantor) +ATLAS-1195 Clean up DSL Translation (jnhagelberg via dkantor) ATLAS-1139 Parameter name of a HDFS DataSet entity should contain filesystem path (svimal2106 via sumasai) ATLAS-1200 Error Catalog enhancement (apoorvnaik via sumasai) ATLAS-1207 Dataset exists query in lineage APIs takes longer (shwethags) @@ -161,7 +162,7 @@ ATLAS-1226 Servlet init-params in web.xml are unused (mneethiraj via shwethags) ATLAS-1224 Minor fixes for hive and falcon bridge twiki (ayubkhan via sumasai) ATLAS-1223 Type REST API v2 implementation (apoorvnaik via sumasai) ATLAS-1210 patch file to add "position" attribute to hive_column type ([email protected] via sumasai) -ATLAS-694 Update Atlas code to use graph abstraction layer (jnhagelb via sumasai) +ATLAS-694 Update Atlas code to use graph abstraction layer (jnhagelberg via sumasai) ATLAS-1215 Atlas UI not working in firefox due to fix in ATLAS-1199 (kevalbhatt) ATLAS-1171 Structured, high-level public APIs - Fix JAXB issues with PList, SearchFilter (mneethiraj via sumasai) ATLAS-1206 Atlas UI not working with IE or Chrome on Windows OS in Kerberos mode (nixonrodrigues via sumasai) @@ -195,7 +196,7 @@ ATLAS-1121 NPE while submitting topology in StormHook (ayubkhan via sumasai) ATLAS-1119 Add retries for edge label creation (sumasai via shwethags) ATLAS-1111 Data loss is observed when atlas is restarted while hive_table metadata ingestion into kafka topic is in-progress(shwethags via sumasai) ATLAS-1115 Show Tag / Taxonomy Listing in sorted order (Kalyanikashikar via sumasai) -ATLAS-1117 Atlas start fails on trunk (jnhagelb via dkantor) +ATLAS-1117 Atlas start fails on trunk (jnhagelberg via dkantor) ATLAS-1112 Hive table GET response from atlas server had duplicate column entries ( ayubkhan, mneethiraj via sumasai) ATLAS-1108 In Atlas HA mode , import-hive.sh in Passive instance fails. (ayubkhan via sumasai) ATLAS-991 Lower bound checking not always disabled for entities being deleted (dkantor) @@ -203,7 +204,7 @@ ATLAS-1104 Get outgoing edges by label doesn't work in some cases (shwethags) ATLAS-1106 Fix Build failure due to wrong version in graphdb/common pom (sumasai) ATLAS-1105 Disable HiveLiteralRewriterTest since its not used currently (sumasai) ATLAS-1103 : UI: Search type list is not refreshed (Kalyanikashikar via sumasai) -ATLAS-693 Titan 0.5.4 implementation of the graph db abstraction (jnhagelb via dkantor) +ATLAS-693 Titan 0.5.4 implementation of the graph db abstraction (jnhagelberg via dkantor) ATLAS-1099 UI : multiple tag assign button hides wrongly (Kalyanikashikar via sumasai) ATLAS-1087 Provide an option to turn off persisting entity definition in audits (sumasai, shwethags) ATLAS-1097 Fix a potential NPE issue flagged by Coverity scan (mneethiraj via shwethags) @@ -218,7 +219,7 @@ ATLAS-1088 Fix /search api to default to fulltext on dsl failure (sumasai) ATLAS-762 Assertion in NegativeSSLAndKerberosTest.testUnsecuredClient needs to be hardened (nixonrodrigues via sumasai) ATLAS-1071 Regression - UI - Details Button under Audits Tab is not working.(kevalbhatt18 via sumasai) ATLAS-965 Old lineage still exists after dropping tables and re-creating tables with same name. (shwethags via sumasai) -ATLAS-1048 TestMetadata.py test in distro project fails on Windows (jnhagelb via shwethags) +ATLAS-1048 TestMetadata.py test in distro project fails on Windows (jnhagelberg via shwethags) ATLAS-1026 StoreBackedTypeCache issues (dkantor via shwethags) ATLAS-861 1 table out of 50,000 tables is left unimported throwing exception during deserialization (sumasai via shwethags) ATLAS-1065 UI: Full text search view same as DSL's (kevalbhat18 via shwethags) @@ -248,7 +249,7 @@ ATLAS-957 Atlas is not capturing topologies that have $ in the data payload (shw ATLAS-1032 Atlas hook package should not include libraries already present in host component - like log4j (mneethiraj via sumasai) ATLAS-1027 Atlas hooks should use properties from atlas-application.properties, instead of component's configuration (mneethiraj via sumasai) ATLAS-1030 Add instrumentation to measure performance: REST API (mneethiraj via sumasai) -ATLAS-996 DSL queries with comparsions of many primitive types fail (jnhagelb via shwethags) +ATLAS-996 DSL queries with comparsions of many primitive types fail (jnhagelberg via shwethags) ATLAS-971 UI not displaying results for this query - Eg: "hive_table as t where qualifiedName = 'default.input@cl1' select t" (kevalbhatt18 via shwethags) ATLAS-1010 Atlas allows recreation of tags with same name (shwethags) ATLAS-990 Hive Import metadata script fails with auth exception (nixonrodrigues via shwethags) @@ -354,7 +355,7 @@ ATLAS-794 Business Catalog Update (jspeidel via yhemanth) ATLAS-837 Enhance Sqoop addon to handle export operation (venkatnrangan via shwethags) ATLAS-869 Make LDAP/AD properties to be configurable. (nixonrodrigues via yhemanth) ATLAS-730 Change titan hbase table name (svimal2106 via sumasai ) -ATLAS-871 Make audit repository implementation configurable (jnhagelb via shwethags) +ATLAS-871 Make audit repository implementation configurable (jnhagelberg via shwethags) ATLAS-885 optimize HBaseStoreManager to avoid expensive HTable instantiation every 5 seconds (madhan.neethiraj via yhemanth) ATLAS-878 UI: Not showing details of SD, DB and COLUMNS (saqeeb.s via shwethags) ATLAS-853 User's name to be mentioned in the top user drop down (saqeeb.s via shwethags) @@ -399,7 +400,7 @@ ATLAS-629 Kafka messages in ATLAS_HOOK might be lost in HA mode at the instant o ATLAS-758 hdfs location of hive table is pointing to old location even after rename ( sumasai ) ATLAS-667 Entity delete should check for required reverse references ( dkantor via sumasai ) ATLAS-738 Add query ability on system properties like guid, state, createdtime etc (shwethags) -ATLAS-692 Create abstraction layer for graph databases (jnhagelb via yhemanth) +ATLAS-692 Create abstraction layer for graph databases (jnhagelberg via yhemanth) ATLAS-689 Migrate Atlas-Storm integration to use Storm 1.0 dependencies. (svimal2106 via yhemanth) ATLAS-754 InstanceSerialization does not serialize Reference in the values array of Reference.(harishjp via sumasai) ATLAS-626 Hive temporary table metadata is captured in atlas (sumasai) http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/b63d6316/webapp/src/main/java/org/apache/atlas/web/filters/StaleTransactionCleanupFilter.java ---------------------------------------------------------------------- diff --git a/webapp/src/main/java/org/apache/atlas/web/filters/StaleTransactionCleanupFilter.java b/webapp/src/main/java/org/apache/atlas/web/filters/StaleTransactionCleanupFilter.java new file mode 100644 index 0000000..5f62bc4 --- /dev/null +++ b/webapp/src/main/java/org/apache/atlas/web/filters/StaleTransactionCleanupFilter.java @@ -0,0 +1,63 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.atlas.web.filters; + +import java.io.IOException; + +import javax.servlet.Filter; +import javax.servlet.FilterChain; +import javax.servlet.FilterConfig; +import javax.servlet.ServletException; +import javax.servlet.ServletRequest; +import javax.servlet.ServletResponse; + +import org.apache.atlas.repository.graph.AtlasGraphProvider; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.inject.Singleton; + +/** + * Filter that rolls back the stale transaction associated with + * the thread running this request so that the request will + * see all of the committed changes. + * + */ +@Singleton +public class StaleTransactionCleanupFilter implements Filter { + + private static final Logger LOG = LoggerFactory.getLogger(StaleTransactionCleanupFilter.class); + + @Override + public void init(FilterConfig filterConfig) throws ServletException { + //nothing to do + } + + @Override + public void doFilter(ServletRequest request, ServletResponse response, FilterChain filterChain) + throws IOException, ServletException { + AtlasGraphProvider.getGraphInstance().rollback(); + filterChain.doFilter(request, response); + } + + @Override + public void destroy() { + // do nothing + } +} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/b63d6316/webapp/src/main/java/org/apache/atlas/web/listeners/GuiceServletConfig.java ---------------------------------------------------------------------- diff --git a/webapp/src/main/java/org/apache/atlas/web/listeners/GuiceServletConfig.java b/webapp/src/main/java/org/apache/atlas/web/listeners/GuiceServletConfig.java index d0437fc..95647e0 100755 --- a/webapp/src/main/java/org/apache/atlas/web/listeners/GuiceServletConfig.java +++ b/webapp/src/main/java/org/apache/atlas/web/listeners/GuiceServletConfig.java @@ -35,6 +35,7 @@ import org.apache.atlas.repository.graphdb.AtlasGraph; import org.apache.atlas.service.Services; import org.apache.atlas.web.filters.ActiveServerFilter; import org.apache.atlas.web.filters.AuditFilter; +import org.apache.atlas.web.filters.StaleTransactionCleanupFilter; import org.apache.atlas.web.service.ActiveInstanceElectorModule; import org.apache.atlas.web.service.ServiceModule; import org.apache.commons.configuration.Configuration; @@ -91,6 +92,7 @@ public class GuiceServletConfig extends GuiceServletContextListener { @Override protected void configureServlets() { + filter("/*").through(StaleTransactionCleanupFilter.class); filter("/*").through(AuditFilter.class); configureActiveServerFilterIfNecessary(); http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/b63d6316/webapp/src/main/java/org/apache/atlas/web/resources/BaseService.java ---------------------------------------------------------------------- diff --git a/webapp/src/main/java/org/apache/atlas/web/resources/BaseService.java b/webapp/src/main/java/org/apache/atlas/web/resources/BaseService.java index fb77b11..2e999fc 100644 --- a/webapp/src/main/java/org/apache/atlas/web/resources/BaseService.java +++ b/webapp/src/main/java/org/apache/atlas/web/resources/BaseService.java @@ -52,7 +52,6 @@ public abstract class BaseService { protected Result getResource(ResourceProvider provider, Request request) throws ResourceNotFoundException { - initializeGraphTransaction(); try { return provider.getResourceById(request); } catch (RuntimeException e) { @@ -63,7 +62,6 @@ public abstract class BaseService { protected Result getResources(ResourceProvider provider, Request request) throws ResourceNotFoundException, InvalidQueryException { - initializeGraphTransaction(); try { return provider.getResources(request); } catch (RuntimeException e) { @@ -72,7 +70,6 @@ public abstract class BaseService { } protected void createResource(ResourceProvider provider, Request request) throws CatalogException { - initializeGraphTransaction(); try { provider.createResource(request); } catch (RuntimeException e) { @@ -81,7 +78,6 @@ public abstract class BaseService { } protected void updateResource(ResourceProvider provider, Request request) throws CatalogException { - initializeGraphTransaction(); try { provider.updateResourceById(request); } catch (RuntimeException e) { @@ -90,7 +86,6 @@ public abstract class BaseService { } protected void deleteResource(ResourceProvider provider, Request request) throws CatalogException { - initializeGraphTransaction(); try { provider.deleteResourceById(request); @@ -100,7 +95,6 @@ public abstract class BaseService { } protected Collection<String> createResources(ResourceProvider provider, Request request) throws CatalogException { - initializeGraphTransaction(); try { return provider.createResources(request); } catch (RuntimeException e) { @@ -139,11 +133,6 @@ public abstract class BaseService { return serializer; } - //todo: abstract via AtlasTypeSystem - // ensure that the thread wasn't re-pooled with an existing transaction - protected void initializeGraphTransaction() { - AtlasGraphProvider.getGraphInstance().rollback(); - } private RuntimeException wrapRuntimeException(RuntimeException e) { return e instanceof CatalogRuntimeException ? e : new CatalogRuntimeException(e); http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/b63d6316/webapp/src/test/java/org/apache/atlas/web/resources/TaxonomyServiceTest.java ---------------------------------------------------------------------- diff --git a/webapp/src/test/java/org/apache/atlas/web/resources/TaxonomyServiceTest.java b/webapp/src/test/java/org/apache/atlas/web/resources/TaxonomyServiceTest.java index e1734e4..45df9c7 100644 --- a/webapp/src/test/java/org/apache/atlas/web/resources/TaxonomyServiceTest.java +++ b/webapp/src/test/java/org/apache/atlas/web/resources/TaxonomyServiceTest.java @@ -18,22 +18,39 @@ package org.apache.atlas.web.resources; -import org.apache.atlas.AtlasException; -import org.apache.atlas.catalog.*; -import org.apache.atlas.services.MetadataService; -import org.easymock.Capture; -import org.testng.annotations.Test; - -import static org.easymock.EasyMock.*; +import static org.easymock.EasyMock.capture; +import static org.easymock.EasyMock.createNiceMock; +import static org.easymock.EasyMock.createStrictMock; +import static org.easymock.EasyMock.expect; +import static org.easymock.EasyMock.newCapture; +import static org.easymock.EasyMock.replay; +import static org.easymock.EasyMock.verify; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNull; import static org.testng.Assert.assertTrue; +import java.net.URI; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; + import javax.ws.rs.core.PathSegment; import javax.ws.rs.core.Response; import javax.ws.rs.core.UriInfo; -import java.net.URI; -import java.util.*; + +import org.apache.atlas.AtlasException; +import org.apache.atlas.catalog.AtlasTypeSystem; +import org.apache.atlas.catalog.JsonSerializer; +import org.apache.atlas.catalog.Request; +import org.apache.atlas.catalog.ResourceProvider; +import org.apache.atlas.catalog.Result; +import org.apache.atlas.catalog.TaxonomyResourceProvider; +import org.apache.atlas.catalog.TermPath; +import org.apache.atlas.services.MetadataService; +import org.easymock.Capture; +import org.testng.annotations.Test; /** * Unit tests for TaxonomyService. @@ -557,11 +574,6 @@ public class TaxonomyServiceTest { return testSerializer; } - @Override - protected void initializeGraphTransaction() { - transactionInitialized = true; - } - public boolean wasTransactionInitialized() { return transactionInitialized; }
