This is an automated email from the ASF dual-hosted git repository.
sarath pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/atlas.git
The following commit(s) were added to refs/heads/branch-2.0 by this push:
new f944df2 ATLAS-3518: Added new Atlas Audit Framework with Purge
Operation and Unit Test
f944df2 is described below
commit f944df2b13ff3e1694579d724957bdbee76967da
Author: Sidharth <[email protected]>
AuthorDate: Tue Nov 19 15:36:06 2019 -0800
ATLAS-3518: Added new Atlas Audit Framework with Purge Operation and Unit
Test
Signed-off-by: Sarath Subramanian <[email protected]>
(cherry picked from commit 371f967f605ec72e1d1a4fc2030d008cbede27fa)
---
addons/models/0000-Area0/0010-base_model.json | 97 +++++++++-
.../main/java/org/apache/atlas/AtlasClientV2.java | 39 +++-
dashboardv2/public/js/utils/Enums.js | 5 +-
dashboardv3/public/js/utils/Enums.js | 5 +-
.../apache/atlas/model/audit/AtlasAuditEntry.java | 130 +++++++++++++
.../atlas/model/audit/AuditSearchParameters.java | 148 +++++++++++++++
.../model/instance/EntityMutationResponse.java | 31 ++--
.../atlas/repository/audit/AtlasAuditService.java | 188 +++++++++++++++++++
.../atlas/repository/ogm/AtlasAuditEntryDTO.java | 110 +++++++++++
.../repository/store/graph/v1/DeleteHandlerV1.java | 79 ++++----
.../test/java/org/apache/atlas/TestModules.java | 8 +-
.../apache/atlas/glossary/GlossaryServiceTest.java | 4 +-
.../org/apache/atlas/query/BasicTestSetup.java | 2 +-
...xportImportTestBase.java => AtlasTestBase.java} | 15 +-
.../atlas/repository/audit/AdminPurgeTest.java | 168 +++++++++++++++++
.../repository/audit/AtlasAuditServiceTest.java | 162 +++++++++++++++++
.../repository/impexp/AtlasServerServiceTest.java | 2 +-
.../impexp/ExportImportAuditServiceTest.java | 5 +-
.../repository/impexp/ExportIncrementalTest.java | 5 +-
.../atlas/repository/impexp/ExportServiceTest.java | 5 +-
.../repository/impexp/ExportSkipLineageTest.java | 7 +-
.../impexp/HdfsPathEntityCreatorTest.java | 5 +-
.../impexp/ImportReactivateTableTest.java | 7 +-
.../atlas/repository/impexp/ImportServiceTest.java | 9 +-
.../impexp/ImportTransformsShaperTest.java | 5 +-
.../IncrementalExportEntityProviderTest.java | 3 +-
.../RelationshipAttributesExtractorTest.java | 5 +-
.../impexp/ReplicationEntityAttributeTest.java | 5 +-
.../StartEntityFetchByExportRequestTest.java | 3 +-
.../TableReplicationRequestProcessorTest.java | 3 +-
.../impexp/ZipFileResourceTestUtils.java | 201 ---------------------
.../repository/migration/MigrationBaseAsserts.java | 2 +-
.../migration/RelationshipCacheGeneratorTest.java | 2 +-
.../graph/v2/AtlasNamespaceDefStoreV2Test.java | 5 +-
.../ClassificationPropagationTest.java | 2 +-
.../userprofile/UserProfileServiceTest.java | 2 +-
.../apache/atlas/services/MetricsServiceTest.java | 4 +-
.../TestLoadModelUtils.java} | 164 +----------------
.../audit-search-parameter-purge.json | 14 ++
.../audit-search-parameter-without-filter.json | 5 +
.../main/java/org/apache/atlas/RequestContext.java | 16 --
.../apache/atlas/web/resources/AdminResource.java | 46 ++++-
.../atlas/web/adapters/TestEntitiesREST.java | 2 +-
.../web/integration/EntityV2JerseyResourceIT.java | 20 +-
.../atlas/web/resources/AdminResourceTest.java | 4 +-
.../json/audit-search-parameter-purge.json | 24 +++
46 files changed, 1271 insertions(+), 502 deletions(-)
diff --git a/addons/models/0000-Area0/0010-base_model.json
b/addons/models/0000-Area0/0010-base_model.json
index 6bdd2f7..351e07b 100644
--- a/addons/models/0000-Area0/0010-base_model.json
+++ b/addons/models/0000-Area0/0010-base_model.json
@@ -1,5 +1,34 @@
{
- "enumDefs": [],
+ "enumDefs": [
+ {
+ "name": "atlas_operation",
+ "description": "Defines audit operations in Atlas",
+ "typeVersion": "1.0",
+ "serviceType": "atlas_core",
+ "elementDefs": [
+ {
+ "ordinal": 0,
+ "value": "OTHERS"
+ },
+ {
+ "ordinal": 1,
+ "value": "PURGE"
+ },
+ {
+ "ordinal": 2,
+ "value": "EXPORT"
+ },
+ {
+ "ordinal": 3,
+ "value": "IMPORT"
+ },
+ {
+ "ordinal": 4,
+ "value": "IMPORT_DELETE_REPL"
+ }
+ ]
+ }
+ ],
"structDefs": [],
"classificationDefs": [],
"entityDefs": [
@@ -388,6 +417,72 @@
]
},
{
+ "name": "__AtlasAuditEntry",
+ "serviceType": "atlas_core",
+ "typeVersion": "1.0",
+ "superTypes": [
+ "__internal"
+ ],
+ "attributeDefs": [
+ {
+ "name": "userName",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": true,
+ "isOptional": false,
+ "isUnique": false
+ },
+ {
+ "name": "operation",
+ "typeName": "atlas_operation",
+ "cardinality": "SINGLE",
+ "isIndexable": true,
+ "isOptional": false,
+ "isUnique": false
+ },
+ {
+ "name": "startTime",
+ "typeName": "date",
+ "cardinality": "SINGLE",
+ "isIndexable": true,
+ "isOptional": false,
+ "isUnique": false
+ },
+ {
+ "name": "endTime",
+ "typeName": "date",
+ "cardinality": "SINGLE",
+ "isIndexable": true,
+ "isOptional": true,
+ "isUnique": false
+ },
+ {
+ "name": "clientId",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": true,
+ "isOptional": true,
+ "isUnique": false
+ },
+ {
+ "name": "params",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": true,
+ "isOptional": true,
+ "isUnique": false
+ },
+ {
+ "name": "result",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ }
+ ]
+ },
+ {
"name": "ProcessExecution",
"superTypes": [
"Asset"
diff --git a/client/client-v2/src/main/java/org/apache/atlas/AtlasClientV2.java
b/client/client-v2/src/main/java/org/apache/atlas/AtlasClientV2.java
index 8c0a640..5a19a0b 100644
--- a/client/client-v2/src/main/java/org/apache/atlas/AtlasClientV2.java
+++ b/client/client-v2/src/main/java/org/apache/atlas/AtlasClientV2.java
@@ -17,19 +17,23 @@
*/
package org.apache.atlas;
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
import com.google.common.annotations.VisibleForTesting;
import com.sun.jersey.api.client.WebResource;
import com.sun.jersey.core.util.MultivaluedMapImpl;
import org.apache.atlas.model.SearchFilter;
+import org.apache.atlas.model.audit.AtlasAuditEntry;
+import org.apache.atlas.model.audit.AuditSearchParameters;
import org.apache.atlas.model.discovery.AtlasSearchResult;
import org.apache.atlas.model.discovery.SearchParameters;
import org.apache.atlas.model.instance.AtlasClassification;
import
org.apache.atlas.model.instance.AtlasClassification.AtlasClassifications;
import org.apache.atlas.model.instance.AtlasEntity.AtlasEntitiesWithExtInfo;
import org.apache.atlas.model.instance.AtlasEntity.AtlasEntityWithExtInfo;
+import org.apache.atlas.model.instance.AtlasEntityHeaders;
import org.apache.atlas.model.instance.AtlasRelationship;
import
org.apache.atlas.model.instance.AtlasRelationship.AtlasRelationshipWithExtInfo;
-import org.apache.atlas.model.instance.AtlasEntityHeaders;
import org.apache.atlas.model.instance.EntityMutationResponse;
import org.apache.atlas.model.lineage.AtlasLineageInfo;
import org.apache.atlas.model.lineage.AtlasLineageInfo.LineageDirection;
@@ -47,6 +51,7 @@ import javax.ws.rs.HttpMethod;
import javax.ws.rs.core.Cookie;
import javax.ws.rs.core.MultivaluedMap;
import javax.ws.rs.core.Response;
+import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -65,9 +70,10 @@ public class AtlasClientV2 extends AtlasBaseClient {
private static final String GET_BY_GUID_TEMPLATE = TYPES_API +
"%s/guid/%s";
private static final String ENTITY_BULK_API = ENTITY_API + "bulk/";
- //Admin Entity Purge
+ //Admin Entity
private static final String ADMIN_API = BASE_URI + "admin/";
private static final String ENTITY_PURGE_API = ADMIN_API + "purge/";
+ private static final String ATLAS_AUDIT_API = ADMIN_API + "audits/";
// Lineage APIs
private static final String LINEAGE_URI = BASE_URI + "v2/lineage/";
@@ -479,11 +485,37 @@ public class AtlasClientV2 extends AtlasBaseClient {
return callAPI(API_V2.UPDATE_RELATIONSHIP, AtlasRelationship.class,
relationship);
}
+ public List<AtlasAuditEntry> getAtlasAuditByOperation(final
AuditSearchParameters auditSearchParameters) throws AtlasServiceException {
+ ArrayNode response = callAPI(API_V2.GET_ATLAS_AUDITS, ArrayNode.class,
auditSearchParameters);
+
+ return extractResults(response, new ExtractOperation<AtlasAuditEntry,
ObjectNode>() {
+ @Override
+ AtlasAuditEntry extractElement(ObjectNode element) {
+ return AtlasType.fromV1Json(element.toString(),
AtlasAuditEntry.class);
+ }
+ });
+ }
+
@Override
protected API formatPathParameters(final API api, final String... params) {
return new API(String.format(api.getPath(), params), api.getMethod(),
api.getExpectedStatus());
}
+ private class ExtractOperation<T, U> {
+ T extractElement(U element) {
+ return (T) element;
+ }
+ }
+
+ private <T, U> List<T> extractResults(ArrayNode jsonResponse,
ExtractOperation<T, U> extractInterafce) {
+ ArrayList<T> resultsList = new ArrayList<>();
+ for (int index = 0; index < jsonResponse.size(); index++) {
+ Object element = jsonResponse.get(index);
+ resultsList.add(extractInterafce.extractElement((U) element));
+ }
+ return resultsList;
+ }
+
private MultivaluedMap<String, String> attributesToQueryParams(Map<String,
String> attributes) {
return attributesToQueryParams(attributes, null);
}
@@ -580,8 +612,9 @@ public class AtlasClientV2 extends AtlasBaseClient {
public static final API_V2 DELETE_RELATIONSHIP_BY_GUID = new
API_V2(RELATIONSHIPS_URI + "guid/", HttpMethod.DELETE,
Response.Status.NO_CONTENT);
public static final API_V2 CREATE_RELATIONSHIP = new
API_V2(RELATIONSHIPS_URI , HttpMethod.POST, Response.Status.OK);
public static final API_V2 UPDATE_RELATIONSHIP = new
API_V2(RELATIONSHIPS_URI , HttpMethod.PUT, Response.Status.OK);
- public static final API_V2 GET_BULK_HEADERS = new API_V2(ENTITY_API +
BULK_HEADERS, HttpMethod.GET, Response.Status.OK);
+ public static final API_V2 GET_BULK_HEADERS = new
API_V2(ENTITY_API + BULK_HEADERS, HttpMethod.GET, Response.Status.OK);
public static final API_V2 UPDATE_BULK_SET_CLASSIFICATIONS = new
API_V2(ENTITY_API + AtlasClientV2.BULK_SET_CLASSIFICATIONS, HttpMethod.POST,
Response.Status.OK);
+ public static final API_V2 GET_ATLAS_AUDITS = new
API_V2(ATLAS_AUDIT_API, HttpMethod.POST, Response.Status.OK);
private API_V2(String path, String method, Response.Status status) {
super(path, method, status);
diff --git a/dashboardv2/public/js/utils/Enums.js
b/dashboardv2/public/js/utils/Enums.js
index 82e11ca..e5cba08 100644
--- a/dashboardv2/public/js/utils/Enums.js
+++ b/dashboardv2/public/js/utils/Enums.js
@@ -37,7 +37,8 @@ define(['require'], function(require) {
TERM_ADD: "Term Added",
TERM_DELETE: "Term Deleted",
LABEL_ADD: "Label(s) Added",
- LABEL_DELETE: "Label(s) Deleted"
+ LABEL_DELETE: "Label(s) Deleted",
+ ENTITY_PURGE: "Entity Purged"
}
Enums.entityStateReadOnly = {
@@ -211,4 +212,4 @@ define(['require'], function(require) {
1: "true"
};
return Enums;
-});
\ No newline at end of file
+});
diff --git a/dashboardv3/public/js/utils/Enums.js
b/dashboardv3/public/js/utils/Enums.js
index 82e11ca..e5cba08 100644
--- a/dashboardv3/public/js/utils/Enums.js
+++ b/dashboardv3/public/js/utils/Enums.js
@@ -37,7 +37,8 @@ define(['require'], function(require) {
TERM_ADD: "Term Added",
TERM_DELETE: "Term Deleted",
LABEL_ADD: "Label(s) Added",
- LABEL_DELETE: "Label(s) Deleted"
+ LABEL_DELETE: "Label(s) Deleted",
+ ENTITY_PURGE: "Entity Purged"
}
Enums.entityStateReadOnly = {
@@ -211,4 +212,4 @@ define(['require'], function(require) {
1: "true"
};
return Enums;
-});
\ No newline at end of file
+});
diff --git
a/intg/src/main/java/org/apache/atlas/model/audit/AtlasAuditEntry.java
b/intg/src/main/java/org/apache/atlas/model/audit/AtlasAuditEntry.java
new file mode 100644
index 0000000..f2a8b3e
--- /dev/null
+++ b/intg/src/main/java/org/apache/atlas/model/audit/AtlasAuditEntry.java
@@ -0,0 +1,130 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.model.audit;
+
+import com.fasterxml.jackson.annotation.JsonAutoDetect;
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.databind.annotation.JsonSerialize;
+import org.apache.atlas.model.AtlasBaseModelObject;
+
+import java.io.Serializable;
+import java.util.Date;
+
+import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.NONE;
+import static
com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.PUBLIC_ONLY;
+
+@JsonAutoDetect(getterVisibility = PUBLIC_ONLY, setterVisibility =
PUBLIC_ONLY, fieldVisibility = NONE)
+@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class AtlasAuditEntry extends AtlasBaseModelObject implements
Serializable {
+ private static final long serialVersionUID = 1L;
+
+ public enum AuditOperation {
+ PURGE,
+ EXPORT,
+ IMPORT,
+ IMPORT_DELETE_REPL
+ }
+
+ private String userName;
+ private AuditOperation operation;
+ private String params;
+ private Date startTime;
+ private Date endTime;
+ private String clientId;
+ private String result;
+
+ public AtlasAuditEntry() {
+ }
+
+ public AtlasAuditEntry(AuditOperation operation, String userName, String
clientId) {
+ this.operation = operation;
+ this.userName = userName;
+ this.clientId = clientId;
+ }
+
+ public AuditOperation getOperation() {
+ return operation;
+ }
+
+ public void setOperation(AuditOperation operation) {
+ this.operation = operation;
+ }
+
+ public void setUserName(String userName) {
+ this.userName = userName;
+ }
+
+ public String getUserName() {
+ return this.userName;
+ }
+
+ public void setParams(String params) {
+ this.params = params;
+ }
+
+ public String getParams() {
+ return this.params;
+ }
+
+ public void setStartTime(Date startTime) {
+ this.startTime = startTime;
+ }
+
+ public Date getStartTime() {
+ return startTime;
+ }
+
+ public void setEndTime(Date endTime) {
+ this.endTime = endTime;
+ }
+
+ public Date getEndTime() {
+ return this.endTime;
+ }
+
+ public String getClientId() {
+ return this.clientId;
+ }
+
+ public void setClientId(String clientId) {
+ this.clientId = clientId;
+ }
+
+ public String getResult() {
+ return result;
+ }
+
+ public void setResult(String result) {
+ this.result = result;
+ }
+
+ @Override
+ public StringBuilder toString(StringBuilder sb) {
+ sb.append(", userName: ").append(userName);
+ sb.append(", operation: ").append(operation);
+ sb.append(", params: ").append(params);
+ sb.append(", clientId: ").append(clientId);
+ sb.append(", startTime: ").append(startTime);
+ sb.append(", endTime: ").append(endTime);
+ sb.append(", result: ").append(result);
+
+ return sb;
+ }
+}
diff --git
a/intg/src/main/java/org/apache/atlas/model/audit/AuditSearchParameters.java
b/intg/src/main/java/org/apache/atlas/model/audit/AuditSearchParameters.java
new file mode 100644
index 0000000..9120062
--- /dev/null
+++ b/intg/src/main/java/org/apache/atlas/model/audit/AuditSearchParameters.java
@@ -0,0 +1,148 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.model.audit;
+
+import com.fasterxml.jackson.annotation.JsonAutoDetect;
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.databind.annotation.JsonSerialize;
+import org.apache.atlas.SortOrder;
+import org.apache.atlas.model.discovery.SearchParameters.FilterCriteria;
+
+import java.io.Serializable;
+import java.util.Objects;
+
+import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.NONE;
+import static
com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.PUBLIC_ONLY;
+
+@JsonAutoDetect(getterVisibility = PUBLIC_ONLY, setterVisibility =
PUBLIC_ONLY, fieldVisibility = NONE)
+@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class AuditSearchParameters implements Serializable {
+ private static final long serialVersionUID = 1L;
+
+ public static long getSerialVersionUID() {
+ return serialVersionUID;
+ }
+
+ AuditSearchParameters() {}
+
+ private FilterCriteria auditFilters;
+ private int limit;
+ private int offset;
+ private String sortBy;
+ private SortOrder sortOrder;
+
+ /**
+ * Entity attribute filters for the type (if type name is specified)
+ * @return
+ */
+ public FilterCriteria getAuditFilters() {
+ return auditFilters;
+ }
+
+ /**
+ * Filter the entities on this criteria
+ * @param auditFilters
+ */
+ public void setAuditFilters(FilterCriteria auditFilters) {
+ this.auditFilters = auditFilters;
+ }
+
+ /**
+ * @return Max number of results to be returned
+ */
+ public int getLimit() {
+ return limit;
+ }
+
+ /**
+ * Restrict the results to the specified limit
+ * @param limit max number of results
+ */
+ public void setLimit(int limit) {
+ this.limit = limit;
+ }
+
+ /**
+ * @return Offset(pagination) of the results
+ */
+ public int getOffset() {
+ return offset;
+ }
+
+ /**
+ * @param offset
+ */
+ public void setOffset(int offset) {
+ this.offset = offset;
+ }
+
+ /**
+ * @return Attribute on which to sort the results
+ */
+ public String getSortBy() { return sortBy; }
+
+ /**
+ * Sort the results based on sortBy attribute
+ * @param sortBy Attribute on which to sort the results
+ */
+ public void setSortBy(String sortBy) { this.sortBy = sortBy; }
+
+ /**
+ * @return Sorting order of the results
+ */
+ public SortOrder getSortOrder() {
+ return sortOrder;
+ }
+
+ /**
+ * Sorting order to sort the results
+ * @param sortOrder ASCENDING vs DESCENDING
+ */
+ public void setSortOrder(SortOrder sortOrder) { this.sortOrder =
sortOrder; }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ AuditSearchParameters that = (AuditSearchParameters) o;
+ return Objects.equals(auditFilters, that.auditFilters) &&
+ limit == that.limit &&
+ offset == that.offset &&
+ Objects.equals(sortBy, that.sortBy) &&
+ Objects.equals(sortOrder, that.sortOrder);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(auditFilters, limit, offset, sortBy, sortOrder);
+ }
+
+ @Override
+ public String toString() {
+ final StringBuilder sb = new StringBuilder("AuditSearchParameters{");
+ sb.append("auditFilters=").append(auditFilters);
+ sb.append(", limit=").append(limit);
+ sb.append(", offset=").append(offset);
+ sb.append(", sortBy='").append(sortBy).append('\'');
+ sb.append(", sortOrder=").append(sortOrder);
+ sb.append('}');
+ return sb.toString();
+ }
+}
diff --git
a/intg/src/main/java/org/apache/atlas/model/instance/EntityMutationResponse.java
b/intg/src/main/java/org/apache/atlas/model/instance/EntityMutationResponse.java
index b448d51..1434a24 100644
---
a/intg/src/main/java/org/apache/atlas/model/instance/EntityMutationResponse.java
+++
b/intg/src/main/java/org/apache/atlas/model/instance/EntityMutationResponse.java
@@ -22,21 +22,16 @@ import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.databind.annotation.JsonSerialize;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
+import org.apache.atlas.model.instance.EntityMutations.EntityOperation;
+import org.apache.atlas.model.typedef.AtlasBaseTypeDef;
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.lang.StringUtils;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlRootElement;
-
-import org.apache.atlas.model.typedef.AtlasBaseTypeDef;
-import org.apache.atlas.model.instance.EntityMutations.EntityOperation;
-import org.apache.commons.collections.CollectionUtils;
-import org.apache.commons.lang.StringUtils;
+import java.util.*;
+import java.util.stream.Collectors;
import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.NONE;
import static
com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.PUBLIC_ONLY;
@@ -123,6 +118,20 @@ public class EntityMutationResponse {
}
@JsonIgnore
+ public String getPurgedEntitiesIds() {
+ String ret = null;
+ List<AtlasEntityHeader> purgedEntities = getPurgedEntities();
+
+ if (CollectionUtils.isNotEmpty(purgedEntities)) {
+ List<String> entityIds = purgedEntities.stream().map(entity ->
entity.getGuid()).collect(Collectors.toList());
+
+ ret = String.join(",", entityIds);
+ }
+
+ return ret;
+ }
+
+ @JsonIgnore
public AtlasEntityHeader getFirstEntityCreated() {
final List<AtlasEntityHeader> entitiesByOperation =
getEntitiesByOperation(EntityOperation.CREATE);
if ( entitiesByOperation != null && entitiesByOperation.size() > 0) {
diff --git
a/repository/src/main/java/org/apache/atlas/repository/audit/AtlasAuditService.java
b/repository/src/main/java/org/apache/atlas/repository/audit/AtlasAuditService.java
new file mode 100644
index 0000000..590f7a0
--- /dev/null
+++
b/repository/src/main/java/org/apache/atlas/repository/audit/AtlasAuditService.java
@@ -0,0 +1,188 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.repository.audit;
+
+import org.apache.atlas.AtlasErrorCode;
+import org.apache.atlas.annotation.AtlasService;
+import org.apache.atlas.annotation.GraphTransaction;
+import org.apache.atlas.discovery.AtlasDiscoveryService;
+import org.apache.atlas.exception.AtlasBaseException;
+import org.apache.atlas.model.audit.AtlasAuditEntry;
+import org.apache.atlas.model.audit.AtlasAuditEntry.AuditOperation;
+import org.apache.atlas.model.audit.AuditSearchParameters;
+import org.apache.atlas.model.discovery.AtlasSearchResult;
+import org.apache.atlas.model.discovery.SearchParameters;
+import org.apache.atlas.model.instance.AtlasEntityHeader;
+import org.apache.atlas.repository.ogm.AtlasAuditEntryDTO;
+import org.apache.atlas.repository.ogm.DataAccess;
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.inject.Inject;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.List;
+import java.util.Set;
+
+@AtlasService
+public class AtlasAuditService {
+ private static final Logger LOG =
LoggerFactory.getLogger(AtlasAuditService.class);
+ private static final String ENTITY_TYPE_AUDIT_ENTRY = "__AtlasAuditEntry";
+
+ private final DataAccess dataAccess;
+ private final AtlasDiscoveryService discoveryService;
+
+ @Inject
+ public AtlasAuditService(DataAccess dataAccess, AtlasDiscoveryService
discoveryService) {
+ this.dataAccess = dataAccess;
+ this.discoveryService = discoveryService;
+ }
+
+ @GraphTransaction
+ public void save(AtlasAuditEntry entry) throws AtlasBaseException {
+ dataAccess.saveNoLoad(entry);
+ }
+
+ public void add(String userName, AuditOperation operation, String
clientId, Date startTime,
+ Date endTime, String params, String result) throws
AtlasBaseException {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("==> AtlasAuditService.add()");
+ }
+
+ AtlasAuditEntry entry = new AtlasAuditEntry();
+
+ entry.setUserName(userName);
+ entry.setOperation(operation);
+ entry.setClientId(clientId);
+ entry.setStartTime(startTime);
+ entry.setEndTime(endTime);
+ entry.setParams(params);
+ entry.setResult(result);
+
+ save(entry);
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("addAuditEntry: user: {}, clientId: {}, operation: {} ",
entry.getUserName(),
+ entry.getClientId(), entry.getOperation());
+ LOG.debug("<== AtlasAuditService.add({})");
+ }
+ }
+
+ public AtlasAuditEntry get(AtlasAuditEntry entry) throws
AtlasBaseException {
+ if(entry.getGuid() == null) {
+ throw new AtlasBaseException("Entity does not have GUID set. load
cannot proceed.");
+ }
+ return dataAccess.load(entry);
+ }
+
+ public List<AtlasAuditEntry> get(AuditSearchParameters
auditSearchParameters) throws AtlasBaseException {
+ if (auditSearchParameters == null) {
+ throw new AtlasBaseException(AtlasErrorCode.INVALID_PARAMETERS,
"Audit Search Parameters not specified");
+ }
+
+ SearchParameters searchParameters =
getSearchParameters(auditSearchParameters);
+
+ searchParameters.setAttributes(getAuditEntityAttributes());
+
+ AtlasSearchResult result =
discoveryService.searchWithParameters(searchParameters);
+ return toAtlasAuditEntry(result);
+ }
+
+ private Set<String> getAuditEntityAttributes() {
+ return AtlasAuditEntryDTO.getAttributes();
+ }
+
+ private List<AtlasAuditEntry> toAtlasAuditEntry(AtlasSearchResult result) {
+ List<AtlasAuditEntry> ret = new ArrayList<>();
+
+ if(CollectionUtils.isNotEmpty(result.getEntities())) {
+ for (AtlasEntityHeader entityHeader : result.getEntities()) {
+ AtlasAuditEntry entry =
AtlasAuditEntryDTO.from(entityHeader.getGuid(),
+ entityHeader.getAttributes());
+ if (entry == null) {
+ continue;
+ }
+
+ ret.add(entry);
+ }
+ }
+
+ return ret;
+ }
+
+ private SearchParameters getSearchParameters(AuditSearchParameters
auditSearchParameters) throws AtlasBaseException {
+ SearchParameters searchParameters = new SearchParameters();
+ searchParameters.setTypeName(ENTITY_TYPE_AUDIT_ENTRY);
+
+ SearchParameters.FilterCriteria validFilter =
getNonEmptyFilter(auditSearchParameters.getAuditFilters());
+ searchParameters.setEntityFilters(validFilter);
+
+ searchParameters.setLimit(auditSearchParameters.getLimit());
+ searchParameters.setOffset(auditSearchParameters.getOffset());
+
+ String sortBy = auditSearchParameters.getSortBy();
+ validateSortByParameter(sortBy);
+
+ searchParameters.setSortBy(auditSearchParameters.getSortBy());
+ searchParameters.setSortOrder(auditSearchParameters.getSortOrder());
+
+ return searchParameters;
+ }
+
+ private void validateSortByParameter(String sortBy) throws
AtlasBaseException{
+ if (StringUtils.isNotEmpty(sortBy) &&
!AtlasAuditEntryDTO.getAttributes().contains(sortBy)) {
+ throw new AtlasBaseException(AtlasErrorCode.UNKNOWN_ATTRIBUTE,
sortBy, "Atlas Audit Entry");
+ }
+ }
+
+ private SearchParameters.FilterCriteria
getNonEmptyFilter(SearchParameters.FilterCriteria auditFilter) throws
AtlasBaseException {
+ SearchParameters.FilterCriteria outCriteria = new
SearchParameters.FilterCriteria();
+ outCriteria.setCriterion(new ArrayList<>());
+
+ if(auditFilter != null) {
+ outCriteria.setCondition(auditFilter.getCondition());
+ List<SearchParameters.FilterCriteria> givenFilterCriterion =
auditFilter.getCriterion();
+
+ for (SearchParameters.FilterCriteria each : givenFilterCriterion) {
+ if (StringUtils.isNotEmpty(each.getAttributeName()) &&
!AtlasAuditEntryDTO.getAttributes().contains(each.getAttributeName())) {
+ throw new
AtlasBaseException(AtlasErrorCode.UNKNOWN_ATTRIBUTE, each.getAttributeName(),
"Atlas Audit Entry");
+ }
+
+ addParameterIfValueNotEmpty(outCriteria,
each.getAttributeName(), each.getOperator(), each.getAttributeValue());
+ }
+ }
+
+ return outCriteria;
+ }
+
+ private void addParameterIfValueNotEmpty(SearchParameters.FilterCriteria
criteria, String attributeName,
+ SearchParameters.Operator
operator, String value) {
+ if(StringUtils.isNotEmpty(value)) {
+ SearchParameters.FilterCriteria filterCriteria = new
SearchParameters.FilterCriteria();
+ filterCriteria.setAttributeName(attributeName);
+ filterCriteria.setAttributeValue(value);
+ filterCriteria.setOperator(operator);
+
+ criteria.getCriterion().add(filterCriteria);
+ }
+ }
+
+}
\ No newline at end of file
diff --git
a/repository/src/main/java/org/apache/atlas/repository/ogm/AtlasAuditEntryDTO.java
b/repository/src/main/java/org/apache/atlas/repository/ogm/AtlasAuditEntryDTO.java
new file mode 100644
index 0000000..8e4bdb5
--- /dev/null
+++
b/repository/src/main/java/org/apache/atlas/repository/ogm/AtlasAuditEntryDTO.java
@@ -0,0 +1,110 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.repository.ogm;
+
+import org.apache.atlas.exception.AtlasBaseException;
+import org.apache.atlas.model.audit.AtlasAuditEntry;
+import org.apache.atlas.model.instance.AtlasEntity;
+import org.apache.atlas.repository.Constants;
+import org.apache.atlas.type.AtlasTypeRegistry;
+import org.springframework.stereotype.Component;
+
+import javax.inject.Inject;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.Date;
+import java.util.Arrays;
+
+@Component
+public class AtlasAuditEntryDTO extends
AbstractDataTransferObject<AtlasAuditEntry> {
+
+ public static final String ATTRIBUTE_USER_NAME = "userName";
+ public static final String ATTRIBUTE_OPERATION = "operation";
+ public static final String ATTRIBUTE_PARAMS = "params";
+ public static final String ATTRIBUTE_START_TIME = "startTime";
+ public static final String ATTRIBUTE_END_TIME = "endTime";
+ public static final String ATTRIBUTE_CLIENT_ID = "clientId";
+ public static final String ATTRIBUTE_RESULT = "result";
+
+ private static final Set<String> ATTRIBUTE_NAMES = new
HashSet<>(Arrays.asList(ATTRIBUTE_USER_NAME,
+ ATTRIBUTE_OPERATION, ATTRIBUTE_PARAMS,
+ ATTRIBUTE_START_TIME, ATTRIBUTE_END_TIME,
+ ATTRIBUTE_CLIENT_ID, ATTRIBUTE_RESULT));
+
+ @Inject
+ public AtlasAuditEntryDTO(AtlasTypeRegistry typeRegistry) {
+ super(typeRegistry, AtlasAuditEntry.class,
+ Constants.INTERNAL_PROPERTY_KEY_PREFIX +
AtlasAuditEntry.class.getSimpleName());
+ }
+
+ public static Set<String> getAttributes() {
+ return ATTRIBUTE_NAMES;
+ }
+
+ public static AtlasAuditEntry from(String guid, Map<String,Object>
attributes) {
+ AtlasAuditEntry entry = new AtlasAuditEntry();
+
+ entry.setGuid(guid);
+ entry.setUserName((String) attributes.get(ATTRIBUTE_USER_NAME));
+
entry.setOperation(AtlasAuditEntry.AuditOperation.valueOf((String)attributes.get(ATTRIBUTE_OPERATION)));
+ entry.setParams((String) attributes.get(ATTRIBUTE_PARAMS));
+ entry.setStartTime((Date) attributes.get(ATTRIBUTE_START_TIME));
+ entry.setEndTime((Date) attributes.get(ATTRIBUTE_END_TIME));
+ entry.setClientId((String) attributes.get(ATTRIBUTE_CLIENT_ID));
+ entry.setResult((String) attributes.get(ATTRIBUTE_RESULT));
+
+ return entry;
+ }
+
+ @Override
+ public AtlasAuditEntry from(AtlasEntity entity) {
+ return from(entity.getGuid(), entity.getAttributes());
+ }
+
+ @Override
+ public AtlasAuditEntry from(AtlasEntity.AtlasEntityWithExtInfo
entityWithExtInfo) {
+ return from(entityWithExtInfo.getEntity());
+ }
+
+ @Override
+ public AtlasEntity toEntity(AtlasAuditEntry obj) {
+ AtlasEntity entity = getDefaultAtlasEntity(obj);
+
+ entity.setAttribute(ATTRIBUTE_USER_NAME, obj.getUserName());
+ entity.setAttribute(ATTRIBUTE_OPERATION, obj.getOperation());
+ entity.setAttribute(ATTRIBUTE_PARAMS, obj.getParams());
+ entity.setAttribute(ATTRIBUTE_START_TIME, obj.getStartTime());
+ entity.setAttribute(ATTRIBUTE_END_TIME, obj.getEndTime());
+ entity.setAttribute(ATTRIBUTE_CLIENT_ID, obj.getClientId());
+ entity.setAttribute(ATTRIBUTE_RESULT, obj.getResult());
+
+ return entity;
+ }
+
+ @Override
+ public AtlasEntity.AtlasEntityWithExtInfo
toEntityWithExtInfo(AtlasAuditEntry obj) throws AtlasBaseException {
+ return new AtlasEntity.AtlasEntityWithExtInfo(toEntity(obj));
+ }
+
+ @Override
+ public Map<String, Object> getUniqueAttributes(final AtlasAuditEntry obj) {
+ return null;
+ }
+}
diff --git
a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/DeleteHandlerV1.java
b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/DeleteHandlerV1.java
index a87e2b5..7b43860 100644
---
a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/DeleteHandlerV1.java
+++
b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/DeleteHandlerV1.java
@@ -17,7 +17,6 @@
*/
package org.apache.atlas.repository.store.graph.v1;
-
import org.apache.atlas.AtlasErrorCode;
import org.apache.atlas.AtlasException;
import org.apache.atlas.RequestContext;
@@ -25,7 +24,6 @@ import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.TypeCategory;
import org.apache.atlas.model.instance.AtlasClassification;
import org.apache.atlas.model.instance.AtlasEntity;
-import org.apache.atlas.model.instance.AtlasEntity.Status;
import org.apache.atlas.model.instance.AtlasEntityHeader;
import org.apache.atlas.model.instance.AtlasObjectId;
import org.apache.atlas.model.typedef.AtlasRelationshipDef.PropagateTags;
@@ -37,15 +35,9 @@ import
org.apache.atlas.repository.graphdb.AtlasEdgeDirection;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2;
import org.apache.atlas.repository.store.graph.v2.EntityGraphRetriever;
-import org.apache.atlas.type.AtlasArrayType;
-import org.apache.atlas.type.AtlasClassificationType;
-import org.apache.atlas.type.AtlasEntityType;
-import org.apache.atlas.type.AtlasMapType;
-import org.apache.atlas.type.AtlasStructType;
+import org.apache.atlas.type.*;
import org.apache.atlas.type.AtlasStructType.AtlasAttribute;
import
org.apache.atlas.type.AtlasStructType.AtlasAttribute.AtlasRelationshipEdgeDirection;
-import org.apache.atlas.type.AtlasType;
-import org.apache.atlas.type.AtlasTypeRegistry;
import org.apache.atlas.utils.AtlasEntityUtil;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.MapUtils;
@@ -59,20 +51,11 @@ import static org.apache.atlas.model.TypeCategory.*;
import static org.apache.atlas.model.instance.AtlasEntity.Status.ACTIVE;
import static org.apache.atlas.model.instance.AtlasEntity.Status.DELETED;
import static
org.apache.atlas.model.typedef.AtlasRelationshipDef.PropagateTags.ONE_TO_TWO;
-import static
org.apache.atlas.repository.Constants.CLASSIFICATION_EDGE_NAME_PROPERTY_KEY;
-import static
org.apache.atlas.repository.Constants.CLASSIFICATION_ENTITY_STATUS;
-import static org.apache.atlas.repository.Constants.CLASSIFICATION_LABEL;
-import static
org.apache.atlas.repository.Constants.CLASSIFICATION_NAME_DELIMITER;
-import static
org.apache.atlas.repository.Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY;
-import static org.apache.atlas.repository.Constants.MODIFIED_BY_KEY;
-import static
org.apache.atlas.repository.Constants.PROPAGATED_CLASSIFICATION_NAMES_KEY;
-import static
org.apache.atlas.repository.Constants.PROPAGATED_TRAIT_NAMES_PROPERTY_KEY;
-import static
org.apache.atlas.repository.Constants.RELATIONSHIP_GUID_PROPERTY_KEY;
+import static org.apache.atlas.repository.Constants.*;
+import static org.apache.atlas.repository.graph.GraphHelper.getTypeName;
import static org.apache.atlas.repository.graph.GraphHelper.*;
-import static
org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2.getIdFromEdge;
-import static
org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2.getQualifiedAttributePropertyKey;
import static
org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2.getState;
-import static
org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2.isReference;
+import static org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2.*;
import static
org.apache.atlas.type.AtlasStructType.AtlasAttribute.AtlasRelationshipEdgeDirection.OUT;
public abstract class DeleteHandlerV1 {
@@ -105,16 +88,8 @@ public abstract class DeleteHandlerV1 {
for (AtlasVertex instanceVertex : instanceVertices) {
final String guid =
AtlasGraphUtilsV2.getIdFromVertex(instanceVertex);
- final AtlasEntity.Status state = getState(instanceVertex);
- final boolean needToSkip;
-
- if (isPurgeRequested) {
- needToSkip = state == ACTIVE ||
requestContext.isPurgedEntity(guid);
- } else {
- needToSkip = state == DELETED ||
requestContext.isDeletedEntity(guid);
- }
- if (needToSkip) {
+ if (skipVertexForDelete(instanceVertex)) {
if (LOG.isDebugEnabled()) {
if (isPurgeRequested) {
LOG.debug("Skipping purging of entity={} as it is
active or already purged", guid);
@@ -122,7 +97,6 @@ public abstract class DeleteHandlerV1 {
LOG.debug("Skipping deletion of entity={} as it is
already deleted", guid);
}
}
-
continue;
}
@@ -855,23 +829,11 @@ public abstract class DeleteHandlerV1 {
LOG.debug("Removing edge from {} to {} with attribute name {}",
string(outVertex), string(inVertex), attribute.getName());
}
- final RequestContext requestContext = RequestContext.get();
- final String typeName =
GraphHelper.getTypeName(outVertex);
- final String outId = GraphHelper.getGuid(outVertex);
- final Status state = getState(outVertex);
- final boolean needToSkip;
-
- if (requestContext.isPurgeRequested()) {
- needToSkip = state == ACTIVE || (outId != null &&
requestContext.isPurgedEntity(outId));
- } else {
- needToSkip = state == DELETED || (outId != null &&
requestContext.isDeletedEntity(outId));
- }
-
- if (needToSkip) {
+ if (skipVertexForDelete(outVertex)) {
return;
}
- AtlasStructType parentType = (AtlasStructType)
typeRegistry.getType(typeName);
+ AtlasStructType parentType = (AtlasStructType)
typeRegistry.getType(GraphHelper.getTypeName(outVertex));
String propertyName =
getQualifiedAttributePropertyKey(parentType, attribute.getName());
String edgeLabel = attribute.getRelationshipEdgeLabel();
AtlasEdge edge = null;
@@ -960,6 +922,9 @@ public abstract class DeleteHandlerV1 {
if (edge != null) {
deleteEdge(edge, isInternalType(inVertex) &&
isInternalType(outVertex));
+ final RequestContext requestContext = RequestContext.get();
+ final String outId =
GraphHelper.getGuid(outVertex);
+
if (! requestContext.isUpdatedEntity(outId)) {
AtlasGraphUtilsV2.setEncodedProperty(outVertex,
MODIFICATION_TIMESTAMP_PROPERTY_KEY, requestContext.getRequestTime());
AtlasGraphUtilsV2.setEncodedProperty(outVertex,
MODIFIED_BY_KEY, requestContext.getUser());
@@ -1080,4 +1045,28 @@ public abstract class DeleteHandlerV1 {
deleteEdgeReference(edge, CLASSIFICATION, false, false,
instanceVertex);
}
}
+
+ private boolean skipVertexForDelete(AtlasVertex vertex) {
+ boolean ret = true;
+
+ if(vertex != null) {
+ try {
+ final RequestContext reqContext = RequestContext.get();
+ final String guid = AtlasGraphUtilsV2.getIdFromVertex(vertex);
+
+ if(guid != null && !reqContext.isDeletedEntity(guid)) {
+ final AtlasEntity.Status vertexState = getState(vertex);
+ if (reqContext.isPurgeRequested()) {
+ ret = vertexState == ACTIVE; // skip purging ACTIVE
vertices
+ } else {
+ ret = vertexState == DELETED; // skip deleting DELETED
vertices
+ }
+ }
+ } catch (IllegalStateException excp) {
+ LOG.warn("skipVertexForDelete(): failed guid/state for the
vertex", excp);
+ }
+ }
+
+ return ret;
+ }
}
diff --git a/repository/src/test/java/org/apache/atlas/TestModules.java
b/repository/src/test/java/org/apache/atlas/TestModules.java
index 94b08a1..06e0ebc 100644
--- a/repository/src/test/java/org/apache/atlas/TestModules.java
+++ b/repository/src/test/java/org/apache/atlas/TestModules.java
@@ -41,16 +41,17 @@ import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.repository.graphdb.GraphDBMigrator;
import
org.apache.atlas.repository.graphdb.janus.migration.GraphDBGraphSONMigrator;
import org.apache.atlas.repository.impexp.ExportService;
+import org.apache.atlas.repository.ogm.AtlasAuditEntryDTO;
import org.apache.atlas.repository.ogm.AtlasServerDTO;
-import org.apache.atlas.repository.ogm.ExportImportAuditEntryDTO;
-import org.apache.atlas.repository.ogm.profiles.AtlasSavedSearchDTO;
-import org.apache.atlas.repository.ogm.profiles.AtlasUserProfileDTO;
import org.apache.atlas.repository.ogm.DTORegistry;
import org.apache.atlas.repository.ogm.DataAccess;
import org.apache.atlas.repository.ogm.DataTransferObject;
+import org.apache.atlas.repository.ogm.ExportImportAuditEntryDTO;
import org.apache.atlas.repository.ogm.glossary.AtlasGlossaryCategoryDTO;
import org.apache.atlas.repository.ogm.glossary.AtlasGlossaryDTO;
import org.apache.atlas.repository.ogm.glossary.AtlasGlossaryTermDTO;
+import org.apache.atlas.repository.ogm.profiles.AtlasSavedSearchDTO;
+import org.apache.atlas.repository.ogm.profiles.AtlasUserProfileDTO;
import org.apache.atlas.repository.store.graph.AtlasEntityStore;
import org.apache.atlas.repository.store.graph.AtlasRelationshipStore;
import org.apache.atlas.repository.store.graph.BulkImporter;
@@ -169,6 +170,7 @@ public class TestModules {
availableDTOs.addBinding().to(AtlasGlossaryCategoryDTO.class);
availableDTOs.addBinding().to(AtlasServerDTO.class);
availableDTOs.addBinding().to(ExportImportAuditEntryDTO.class);
+ availableDTOs.addBinding().to(AtlasAuditEntryDTO.class);
bind(DTORegistry.class).asEagerSingleton();
bind(DataAccess.class).asEagerSingleton();
diff --git
a/repository/src/test/java/org/apache/atlas/glossary/GlossaryServiceTest.java
b/repository/src/test/java/org/apache/atlas/glossary/GlossaryServiceTest.java
index be1698f..759dcdf 100644
---
a/repository/src/test/java/org/apache/atlas/glossary/GlossaryServiceTest.java
+++
b/repository/src/test/java/org/apache/atlas/glossary/GlossaryServiceTest.java
@@ -36,7 +36,7 @@ import org.apache.atlas.model.instance.AtlasRelatedObjectId;
import org.apache.atlas.model.instance.EntityMutationResponse;
import org.apache.atlas.model.typedef.AtlasClassificationDef;
import org.apache.atlas.model.typedef.AtlasTypesDef;
-import org.apache.atlas.repository.impexp.ZipFileResourceTestUtils;
+import org.apache.atlas.utils.TestLoadModelUtils;
import org.apache.atlas.repository.store.graph.AtlasEntityStore;
import org.apache.atlas.repository.store.graph.v2.AtlasEntityStream;
import org.apache.atlas.store.AtlasTypeDefStore;
@@ -93,7 +93,7 @@ public class GlossaryServiceTest {
@BeforeClass
public void setupSampleGlossary() {
try {
- ZipFileResourceTestUtils.loadAllModels("0000-Area0", typeDefStore,
typeRegistry);
+ TestLoadModelUtils.loadAllModels("0000-Area0", typeDefStore,
typeRegistry);
} catch (AtlasBaseException | IOException e) {
throw new SkipException("SubjectArea model loading failed");
}
diff --git
a/repository/src/test/java/org/apache/atlas/query/BasicTestSetup.java
b/repository/src/test/java/org/apache/atlas/query/BasicTestSetup.java
index 02f78b3..9aa554a 100644
--- a/repository/src/test/java/org/apache/atlas/query/BasicTestSetup.java
+++ b/repository/src/test/java/org/apache/atlas/query/BasicTestSetup.java
@@ -36,7 +36,7 @@ import java.util.*;
import java.util.stream.Collectors;
import java.util.stream.Stream;
-import static
org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.loadModelFromJson;
+import static org.apache.atlas.utils.TestLoadModelUtils.loadModelFromJson;
import static org.testng.Assert.fail;
public abstract class BasicTestSetup {
diff --git
a/repository/src/test/java/org/apache/atlas/repository/impexp/ExportImportTestBase.java
b/repository/src/test/java/org/apache/atlas/repository/AtlasTestBase.java
similarity index 86%
rename from
repository/src/test/java/org/apache/atlas/repository/impexp/ExportImportTestBase.java
rename to
repository/src/test/java/org/apache/atlas/repository/AtlasTestBase.java
index 925b2a0..c2668b1 100644
---
a/repository/src/test/java/org/apache/atlas/repository/impexp/ExportImportTestBase.java
+++ b/repository/src/test/java/org/apache/atlas/repository/AtlasTestBase.java
@@ -16,11 +16,12 @@
* limitations under the License.
*/
-package org.apache.atlas.repository.impexp;
+package org.apache.atlas.repository;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.impexp.ExportImportAuditEntry;
import org.apache.atlas.model.instance.AtlasEntity;
+import org.apache.atlas.repository.impexp.ExportImportAuditService;
import org.apache.atlas.repository.store.graph.v2.AtlasEntityStoreV2;
import org.apache.atlas.store.AtlasTypeDefStore;
import org.apache.atlas.type.AtlasTypeRegistry;
@@ -30,15 +31,15 @@ import java.io.IOException;
import java.util.Arrays;
import java.util.List;
-import static
org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.createAtlasEntity;
-import static
org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.loadBaseModel;
-import static
org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.loadEntity;
-import static
org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.loadHiveModel;
+import static org.apache.atlas.utils.TestLoadModelUtils.createAtlasEntity;
+import static org.apache.atlas.utils.TestLoadModelUtils.loadBaseModel;
+import static org.apache.atlas.utils.TestLoadModelUtils.loadEntity;
+import static org.apache.atlas.utils.TestLoadModelUtils.loadHiveModel;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertNotNull;
import static org.testng.Assert.assertTrue;
-public class ExportImportTestBase {
+public class AtlasTestBase {
protected static final String ENTITIES_SUB_DIR = "stocksDB-Entities";
protected static final String DB_GUID =
"1637a33e-6512-447b-ade7-249c8cb5344b";
protected static final String TABLE_GUID =
"df122fc3-5555-40f8-a30f-3090b8a622f8";
@@ -69,7 +70,7 @@ public class ExportImportTestBase {
}
}
- protected void assertAuditEntry(ExportImportAuditService auditService)
throws InterruptedException {
+ protected void assertExportImportAuditEntry(ExportImportAuditService
auditService) throws InterruptedException {
pauseForIndexCreation();
List<ExportImportAuditEntry> result = null;
try {
diff --git
a/repository/src/test/java/org/apache/atlas/repository/audit/AdminPurgeTest.java
b/repository/src/test/java/org/apache/atlas/repository/audit/AdminPurgeTest.java
new file mode 100644
index 0000000..0a2fc39
--- /dev/null
+++
b/repository/src/test/java/org/apache/atlas/repository/audit/AdminPurgeTest.java
@@ -0,0 +1,168 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.repository.audit;
+
+import org.apache.atlas.RequestContext;
+import org.apache.atlas.TestModules;
+import org.apache.atlas.TestUtilsV2;
+import org.apache.atlas.exception.AtlasBaseException;
+import org.apache.atlas.model.audit.AtlasAuditEntry;
+import org.apache.atlas.model.audit.AuditSearchParameters;
+import org.apache.atlas.model.instance.*;
+import org.apache.atlas.model.typedef.AtlasTypesDef;
+import org.apache.atlas.repository.AtlasTestBase;
+import org.apache.atlas.repository.graph.AtlasGraphProvider;
+import
org.apache.atlas.repository.store.bootstrap.AtlasTypeDefStoreInitializer;
+import org.apache.atlas.repository.store.graph.v2.AtlasEntityStoreV2;
+import org.apache.atlas.repository.store.graph.v2.AtlasEntityStream;
+import org.apache.atlas.runner.LocalSolrRunner;
+import org.apache.atlas.store.AtlasTypeDefStore;
+import org.apache.atlas.type.AtlasTypeRegistry;
+import org.apache.atlas.utils.TestResourceFileUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.testng.Assert;
+import org.testng.SkipException;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.BeforeTest;
+import org.testng.annotations.Guice;
+import org.testng.annotations.Test;
+
+import javax.inject.Inject;
+import java.io.IOException;
+import java.util.*;
+import java.util.stream.Collectors;
+
+import static org.apache.atlas.graph.GraphSandboxUtil.useLocalSolr;
+import static org.testng.Assert.assertNotNull;
+import static org.testng.Assert.assertTrue;
+import static org.testng.Assert.fail;
+
+@Guice(modules = TestModules.TestOnlyModule.class)
+public class AdminPurgeTest extends AtlasTestBase {
+ private static final Logger LOG =
LoggerFactory.getLogger(AdminPurgeTest.class);
+ private static final String CLIENT_HOST = "127.0.0.0";
+ private static final String DEFAULT_USER = "Admin";
+ private static final String AUDIT_PARAMETER_RESOURCE_DIR =
"auditSearchParameters";
+
+ @Inject
+ AtlasTypeRegistry typeRegistry;
+
+ @Inject
+ private AtlasTypeDefStore typeDefStore;
+
+ @Inject
+ private AtlasAuditService auditService;
+
+ @Inject
+ private AtlasEntityStoreV2 entityStore;
+
+ @BeforeTest
+ public void setupTest() throws IOException, AtlasBaseException {
+ RequestContext.clear();
+ RequestContext.get().setUser(TestUtilsV2.TEST_USER, null);
+ basicSetup(typeDefStore, typeRegistry);
+ }
+
+ @AfterClass
+ public void clear() throws Exception {
+ Thread.sleep(1000);
+ AtlasGraphProvider.cleanup();
+
+ if (useLocalSolr()) {
+ LocalSolrRunner.stop();
+ }
+ }
+
+ @Test
+ public void testDeleteEntitiesDoesNotLookupDeletedEntity() throws
Exception {
+ AtlasTypesDef sampleTypes = TestUtilsV2.defineDeptEmployeeTypes();
+ AtlasTypesDef typesToCreate =
AtlasTypeDefStoreInitializer.getTypesToCreate(sampleTypes, typeRegistry);
+
+ if (!typesToCreate.isEmpty()) {
+ typeDefStore.createTypesDef(typesToCreate);
+ }
+
+ AtlasEntity.AtlasEntitiesWithExtInfo deptEg2 =
TestUtilsV2.createDeptEg2();
+ AtlasEntityStream entityStream = new AtlasEntityStream(deptEg2);
+ EntityMutationResponse emr = entityStore.createOrUpdate(entityStream,
false);
+ pauseForIndexCreation();
+
+ assertNotNull(emr);
+ assertNotNull(emr.getCreatedEntities());
+ assertTrue(emr.getCreatedEntities().size() > 0);
+
+ List<String> guids = emr.getCreatedEntities().stream()
+ .map(p -> new
String(p.getGuid())).collect(Collectors.toList());
+ EntityMutationResponse response = entityStore.deleteByIds(guids);
+ pauseForIndexCreation();
+
+ List<AtlasEntityHeader> responseDeletedEntities =
response.getDeletedEntities();
+ assertNotNull(responseDeletedEntities);
+
+ responseDeletedEntities.sort((l,r) ->
l.getGuid().compareTo(r.getGuid()));
+ List<AtlasEntityHeader> toBeDeletedEntities = emr.getCreatedEntities();
+ toBeDeletedEntities.sort((l,r) -> l.getGuid().compareTo(r.getGuid()));
+ Assert.assertEquals(responseDeletedEntities.size(),
emr.getCreatedEntities().size());
+ for(int index = 0 ; index < responseDeletedEntities.size(); index++)
+ Assert.assertEquals(responseDeletedEntities.get(index).getGuid(),
emr.getCreatedEntities().get(index).getGuid());
+
+ Date startTimestamp = new Date();
+ response = entityStore.purgeByIds(new HashSet<>(guids));
+ pauseForIndexCreation();
+
+ List<AtlasEntityHeader> responsePurgedEntities =
response.getPurgedEntities();
+ responsePurgedEntities.sort((l,r) ->
l.getGuid().compareTo(r.getGuid()));
+ Assert.assertEquals(responsePurgedEntities.size(),
responseDeletedEntities.size());
+ for(int index = 0 ; index < responsePurgedEntities.size(); index++)
+ Assert.assertEquals(responsePurgedEntities.get(index).getGuid(),
responseDeletedEntities.get(index).getGuid());
+
+ auditService.add(DEFAULT_USER, AtlasAuditEntry.AuditOperation.PURGE,
+ CLIENT_HOST, startTimestamp, new Date(), guids.toString(),
+ response.getPurgedEntitiesIds());
+
+ AuditSearchParameters auditParameterNull =
createAuditParameter("audit-search-parameter-without-filter");
+ assertAuditEntry(auditService, auditParameterNull);
+
+ AuditSearchParameters auditSearchParameters =
createAuditParameter("audit-search-parameter-purge");
+ assertAuditEntry(auditService, auditSearchParameters);
+ }
+
+ private AuditSearchParameters createAuditParameter(String fileName) {
+ try {
+ return
TestResourceFileUtils.readObjectFromJson(AUDIT_PARAMETER_RESOURCE_DIR,
fileName, AuditSearchParameters.class);
+ } catch (IOException e) {
+ fail(e.getMessage());
+ }
+ return null;
+ }
+
+ private void assertAuditEntry(AtlasAuditService auditService,
AuditSearchParameters auditSearchParameters) throws InterruptedException {
+ pauseForIndexCreation();
+ List<AtlasAuditEntry> result = null;
+ try {
+
+ result = auditService.get(auditSearchParameters);
+ } catch (Exception e) {
+ throw new SkipException("audit entries not retrieved.");
+ }
+
+ assertNotNull(result);
+ assertTrue(result.size() > 0);
+ }
+}
diff --git
a/repository/src/test/java/org/apache/atlas/repository/audit/AtlasAuditServiceTest.java
b/repository/src/test/java/org/apache/atlas/repository/audit/AtlasAuditServiceTest.java
new file mode 100644
index 0000000..54f75cc
--- /dev/null
+++
b/repository/src/test/java/org/apache/atlas/repository/audit/AtlasAuditServiceTest.java
@@ -0,0 +1,162 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.repository.audit;
+
+import org.apache.atlas.TestModules;
+import org.apache.atlas.exception.AtlasBaseException;
+import org.apache.atlas.model.audit.AtlasAuditEntry;
+import org.apache.atlas.model.audit.AtlasAuditEntry.AuditOperation;
+import org.apache.atlas.model.audit.AuditSearchParameters;
+import org.apache.atlas.store.AtlasTypeDefStore;
+import org.apache.atlas.type.AtlasType;
+import org.apache.atlas.type.AtlasTypeRegistry;
+import org.apache.atlas.utils.TestResourceFileUtils;
+import org.testng.SkipException;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Guice;
+import org.testng.annotations.Test;
+
+import javax.inject.Inject;
+import java.io.IOException;
+import java.util.Date;
+import java.util.List;
+
+import static org.apache.atlas.utils.TestLoadModelUtils.loadBaseModel;
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertNotEquals;
+import static org.testng.Assert.assertNotNull;
+import static org.testng.Assert.assertTrue;
+import static org.testng.Assert.fail;
+
+@Guice(modules = TestModules.TestOnlyModule.class)
+public class AtlasAuditServiceTest {
+
+ private static final int WAIT_TIME_FOR_INDEX_CREATION_IN_MILLI = 5000;
+
+ private static final String AUDIT_PARAMETER_RESOURCE_DIR =
"auditSearchParameters";
+
+ private static final String DEFAULT_USER = "admin";
+
+ @Inject
+ AtlasTypeRegistry typeRegistry;
+
+ @Inject
+ private AtlasTypeDefStore typeDefStore;
+
+ @Inject
+ AtlasAuditService auditService;
+
+ @BeforeClass
+ public void setup() throws IOException, AtlasBaseException {
+ loadBaseModel(typeDefStore, typeRegistry);
+ }
+
+ @Test
+ public void checkTypeRegistered() throws AtlasBaseException {
+ AtlasType auditEntryType = typeRegistry.getType("__" +
AtlasAuditEntry.class.getSimpleName());
+ assertNotNull(auditEntryType);
+ }
+
+ @Test
+ public void checkStoringOfAuditEntry() throws AtlasBaseException {
+ final String clientId1 = "client1";
+ AtlasAuditEntry entryTobeStored1 = saveEntry(AuditOperation.PURGE,
clientId1);
+
+ String clientId2 = "client2";
+ AtlasAuditEntry entryTobeStored2 = saveEntry(AuditOperation.PURGE,
clientId2);
+
+ waitForIndexCreation();
+ AtlasAuditEntry storedEntry1 = retrieveEntry(entryTobeStored1);
+ AtlasAuditEntry storedEntry2 = retrieveEntry(entryTobeStored2);
+
+ assertNotEquals(storedEntry1.getGuid(), storedEntry2.getGuid());
+
+ assertNotNull(storedEntry1.getGuid());
+ assertNotNull(storedEntry2.getGuid());
+
+ assertEquals(storedEntry1.getUserName(), DEFAULT_USER);
+ assertEquals(storedEntry2.getUserName(), DEFAULT_USER);
+
+ assertEquals(storedEntry1.getClientId(),
entryTobeStored1.getClientId());
+ assertEquals(storedEntry2.getClientId(),
entryTobeStored2.getClientId());
+
+ assertEquals(storedEntry1.getOperation(),
entryTobeStored1.getOperation());
+ assertEquals(storedEntry2.getOperation(),
entryTobeStored2.getOperation());
+ }
+
+ @Test
+ public void checkStoringMultipleAuditEntries() throws AtlasBaseException,
InterruptedException {
+ final String clientId = "client1";
+ final int MAX_ENTRIES = 5;
+ final int LIMIT_PARAM = 3;
+
+ for (int i = 0; i < MAX_ENTRIES; i++) {
+ saveEntry(AuditOperation.PURGE, clientId);
+ }
+
+ waitForIndexCreation();
+ AuditSearchParameters auditSearchParameters =
createAuditParameter("audit-search-parameter-purge");
+ auditSearchParameters.setLimit(LIMIT_PARAM);
+ auditSearchParameters.setOffset(0);
+
+ List<AtlasAuditEntry> resultLimitedByParam =
auditService.get(auditSearchParameters);
+ assertTrue(resultLimitedByParam.size() == LIMIT_PARAM);
+
+ auditSearchParameters.setLimit(MAX_ENTRIES);
+ auditSearchParameters.setOffset(LIMIT_PARAM);
+ List<AtlasAuditEntry> results =
auditService.get(auditSearchParameters);
+ assertTrue(results.size() == (MAX_ENTRIES - LIMIT_PARAM));
+ }
+
+ private AuditSearchParameters createAuditParameter(String fileName) {
+ try {
+ return
TestResourceFileUtils.readObjectFromJson(AUDIT_PARAMETER_RESOURCE_DIR,
fileName, AuditSearchParameters.class);
+ } catch (IOException e) {
+ fail(e.getMessage());
+ }
+ return null;
+ }
+
+ private AtlasAuditEntry retrieveEntry(AtlasAuditEntry entry) throws
AtlasBaseException {
+ AuditSearchParameters auditSearchParameters =
createAuditParameter("audit-search-parameter-purge");
+ AtlasAuditEntry result = auditService.get(entry);
+
+ assertNotNull(result);
+
+ entry.setGuid(result.getGuid());
+ return auditService.get(entry);
+ }
+
+ private AtlasAuditEntry saveEntry(AuditOperation operation, String
clientId) throws AtlasBaseException {
+ AtlasAuditEntry entry = new AtlasAuditEntry(operation, DEFAULT_USER,
clientId);
+
+ entry.setStartTime(new Date());
+ entry.setEndTime(new Date());
+ auditService.save(entry);
+ return entry;
+ }
+
+ protected void waitForIndexCreation() {
+ try {
+ Thread.sleep(WAIT_TIME_FOR_INDEX_CREATION_IN_MILLI);
+ } catch (InterruptedException ex) {
+ throw new SkipException("Wait interrupted.");
+ }
+ }
+}
diff --git
a/repository/src/test/java/org/apache/atlas/repository/impexp/AtlasServerServiceTest.java
b/repository/src/test/java/org/apache/atlas/repository/impexp/AtlasServerServiceTest.java
index 78865c5..91ffc27 100644
---
a/repository/src/test/java/org/apache/atlas/repository/impexp/AtlasServerServiceTest.java
+++
b/repository/src/test/java/org/apache/atlas/repository/impexp/AtlasServerServiceTest.java
@@ -35,7 +35,7 @@ import java.util.HashMap;
import java.util.Map;
import static org.apache.atlas.repository.Constants.ATTR_NAME_REFERENCEABLE;
-import static
org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.loadBaseModel;
+import static org.apache.atlas.utils.TestLoadModelUtils.loadBaseModel;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertNotEquals;
import static org.testng.Assert.assertNotNull;
diff --git
a/repository/src/test/java/org/apache/atlas/repository/impexp/ExportImportAuditServiceTest.java
b/repository/src/test/java/org/apache/atlas/repository/impexp/ExportImportAuditServiceTest.java
index ba7a8a0..52a8174 100644
---
a/repository/src/test/java/org/apache/atlas/repository/impexp/ExportImportAuditServiceTest.java
+++
b/repository/src/test/java/org/apache/atlas/repository/impexp/ExportImportAuditServiceTest.java
@@ -21,6 +21,7 @@ package org.apache.atlas.repository.impexp;
import org.apache.atlas.TestModules;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.impexp.ExportImportAuditEntry;
+import org.apache.atlas.repository.AtlasTestBase;
import org.apache.atlas.store.AtlasTypeDefStore;
import org.apache.atlas.type.AtlasType;
import org.apache.atlas.type.AtlasTypeRegistry;
@@ -32,14 +33,14 @@ import javax.inject.Inject;
import java.io.IOException;
import java.util.List;
-import static
org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.loadBaseModel;
+import static org.apache.atlas.utils.TestLoadModelUtils.loadBaseModel;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertNotEquals;
import static org.testng.Assert.assertNotNull;
import static org.testng.Assert.assertTrue;
@Guice(modules = TestModules.TestOnlyModule.class)
-public class ExportImportAuditServiceTest extends ExportImportTestBase {
+public class ExportImportAuditServiceTest extends AtlasTestBase {
@Inject
AtlasTypeRegistry typeRegistry;
diff --git
a/repository/src/test/java/org/apache/atlas/repository/impexp/ExportIncrementalTest.java
b/repository/src/test/java/org/apache/atlas/repository/impexp/ExportIncrementalTest.java
index 4d43852..0e3955d 100644
---
a/repository/src/test/java/org/apache/atlas/repository/impexp/ExportIncrementalTest.java
+++
b/repository/src/test/java/org/apache/atlas/repository/impexp/ExportIncrementalTest.java
@@ -27,6 +27,7 @@ import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.impexp.AtlasExportRequest;
import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.model.instance.AtlasObjectId;
+import org.apache.atlas.repository.AtlasTestBase;
import org.apache.atlas.repository.store.graph.v2.AtlasEntityStoreV2;
import org.apache.atlas.repository.util.UniqueList;
import org.apache.atlas.store.AtlasTypeDefStore;
@@ -53,7 +54,7 @@ import java.util.List;
import java.util.Map;
import static
org.apache.atlas.model.impexp.AtlasExportRequest.FETCH_TYPE_INCREMENTAL_CHANGE_MARKER;
-import static
org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.createTypes;
+import static org.apache.atlas.utils.TestLoadModelUtils.createTypes;
import static
org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.getEntities;
import static
org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.getZipSource;
import static
org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.runExportWithParameters;
@@ -63,7 +64,7 @@ import static org.testng.Assert.assertNotNull;
import static org.testng.Assert.assertTrue;
@Guice(modules = TestModules.TestOnlyModule.class)
-public class ExportIncrementalTest extends ExportImportTestBase {
+public class ExportIncrementalTest extends AtlasTestBase {
@Inject
AtlasTypeRegistry typeRegistry;
diff --git
a/repository/src/test/java/org/apache/atlas/repository/impexp/ExportServiceTest.java
b/repository/src/test/java/org/apache/atlas/repository/impexp/ExportServiceTest.java
index db8c6c8..8e19dc4 100644
---
a/repository/src/test/java/org/apache/atlas/repository/impexp/ExportServiceTest.java
+++
b/repository/src/test/java/org/apache/atlas/repository/impexp/ExportServiceTest.java
@@ -28,6 +28,7 @@ import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.model.instance.AtlasObjectId;
import org.apache.atlas.model.instance.EntityMutationResponse;
import org.apache.atlas.model.typedef.AtlasTypesDef;
+import org.apache.atlas.repository.AtlasTestBase;
import org.apache.atlas.repository.graph.AtlasGraphProvider;
import
org.apache.atlas.repository.store.bootstrap.AtlasTypeDefStoreInitializer;
import org.apache.atlas.repository.store.graph.v2.AtlasEntityStoreV2;
@@ -61,7 +62,7 @@ import static org.testng.Assert.assertNotNull;
import static org.testng.Assert.assertTrue;
@Guice(modules = TestModules.TestOnlyModule.class)
-public class ExportServiceTest extends ExportImportTestBase {
+public class ExportServiceTest extends AtlasTestBase {
private static final Logger LOG =
LoggerFactory.getLogger(ExportServiceTest.class);
@Inject
@@ -106,7 +107,7 @@ public class ExportServiceTest extends ExportImportTestBase
{
@AfterClass
public void clear() throws Exception {
Thread.sleep(1000);
- assertAuditEntry(auditService);
+ assertExportImportAuditEntry(auditService);
AtlasGraphProvider.cleanup();
if (useLocalSolr()) {
diff --git
a/repository/src/test/java/org/apache/atlas/repository/impexp/ExportSkipLineageTest.java
b/repository/src/test/java/org/apache/atlas/repository/impexp/ExportSkipLineageTest.java
index 25e0a53..f8b7eda7 100644
---
a/repository/src/test/java/org/apache/atlas/repository/impexp/ExportSkipLineageTest.java
+++
b/repository/src/test/java/org/apache/atlas/repository/impexp/ExportSkipLineageTest.java
@@ -25,6 +25,7 @@ import org.apache.atlas.TestUtilsV2;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.impexp.AtlasExportRequest;
import org.apache.atlas.model.instance.AtlasEntity;
+import org.apache.atlas.repository.AtlasTestBase;
import org.apache.atlas.repository.store.graph.v1.DeleteHandlerDelegate;
import org.apache.atlas.repository.store.graph.v2.AtlasEntityChangeNotifier;
import org.apache.atlas.repository.store.graph.v2.AtlasEntityStoreV2;
@@ -43,8 +44,8 @@ import java.io.IOException;
import java.io.InputStream;
import java.util.Map;
-import static
org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.loadBaseModel;
-import static
org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.loadHiveModel;
+import static org.apache.atlas.utils.TestLoadModelUtils.loadBaseModel;
+import static org.apache.atlas.utils.TestLoadModelUtils.loadHiveModel;
import static
org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.runExportWithParameters;
import static org.mockito.Mockito.mock;
import static org.testng.Assert.assertEquals;
@@ -52,7 +53,7 @@ import static org.testng.Assert.assertNotNull;
import static org.testng.AssertJUnit.fail;
@Guice(modules = TestModules.TestOnlyModule.class)
-public class ExportSkipLineageTest extends ExportImportTestBase {
+public class ExportSkipLineageTest extends AtlasTestBase {
@Inject
AtlasTypeRegistry typeRegistry;
diff --git
a/repository/src/test/java/org/apache/atlas/repository/impexp/HdfsPathEntityCreatorTest.java
b/repository/src/test/java/org/apache/atlas/repository/impexp/HdfsPathEntityCreatorTest.java
index 1863b8d..4247f65 100644
---
a/repository/src/test/java/org/apache/atlas/repository/impexp/HdfsPathEntityCreatorTest.java
+++
b/repository/src/test/java/org/apache/atlas/repository/impexp/HdfsPathEntityCreatorTest.java
@@ -20,6 +20,7 @@ package org.apache.atlas.repository.impexp;
import org.apache.atlas.TestModules;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.instance.AtlasEntity;
+import org.apache.atlas.repository.AtlasTestBase;
import org.apache.atlas.store.AtlasTypeDefStore;
import org.apache.atlas.type.AtlasTypeRegistry;
import org.testng.annotations.BeforeClass;
@@ -33,12 +34,12 @@ import static
org.apache.atlas.repository.impexp.HdfsPathEntityCreator.HDFS_PATH
import static
org.apache.atlas.repository.impexp.HdfsPathEntityCreator.HDFS_PATH_ATTRIBUTE_NAME_NAME;
import static
org.apache.atlas.repository.impexp.HdfsPathEntityCreator.HDFS_PATH_ATTRIBUTE_QUALIFIED_NAME;
import static
org.apache.atlas.repository.impexp.HdfsPathEntityCreator.getQualifiedName;
-import static
org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.loadFsModel;
+import static org.apache.atlas.utils.TestLoadModelUtils.loadFsModel;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertNotNull;
@Guice(modules = TestModules.TestOnlyModule.class)
-public class HdfsPathEntityCreatorTest extends ExportImportTestBase {
+public class HdfsPathEntityCreatorTest extends AtlasTestBase {
@Inject
AtlasTypeRegistry typeRegistry;
diff --git
a/repository/src/test/java/org/apache/atlas/repository/impexp/ImportReactivateTableTest.java
b/repository/src/test/java/org/apache/atlas/repository/impexp/ImportReactivateTableTest.java
index d0c06a1..e06bf66 100644
---
a/repository/src/test/java/org/apache/atlas/repository/impexp/ImportReactivateTableTest.java
+++
b/repository/src/test/java/org/apache/atlas/repository/impexp/ImportReactivateTableTest.java
@@ -27,6 +27,7 @@ import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.model.instance.AtlasRelatedObjectId;
import org.apache.atlas.model.instance.AtlasRelationship;
import org.apache.atlas.model.instance.EntityMutationResponse;
+import org.apache.atlas.repository.AtlasTestBase;
import org.apache.atlas.repository.graph.AtlasGraphProvider;
import org.apache.atlas.repository.store.graph.AtlasEntityStore;
import org.apache.atlas.repository.store.graph.v2.AtlasEntityStream;
@@ -48,15 +49,15 @@ import static
org.apache.atlas.AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME;
import static org.apache.atlas.graph.GraphSandboxUtil.useLocalSolr;
import static
org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.getInputStreamFrom;
import static
org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.getDefaultImportRequest;
-import static
org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.loadFsModel;
-import static
org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.loadHiveModel;
+import static org.apache.atlas.utils.TestLoadModelUtils.loadFsModel;
+import static org.apache.atlas.utils.TestLoadModelUtils.loadHiveModel;
import static
org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.runImportWithParameters;
import static org.apache.atlas.type.AtlasTypeUtil.toAtlasRelatedObjectId;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertNotNull;
@Guice(modules = TestModules.TestOnlyModule.class)
-public class ImportReactivateTableTest extends ExportImportTestBase {
+public class ImportReactivateTableTest extends AtlasTestBase {
private static final Logger LOG =
LoggerFactory.getLogger(ImportReactivateTableTest.class);
private static final String ENTITY_TYPE_COL = "hive_column";
diff --git
a/repository/src/test/java/org/apache/atlas/repository/impexp/ImportServiceTest.java
b/repository/src/test/java/org/apache/atlas/repository/impexp/ImportServiceTest.java
index 1cb0140..570b1f8 100644
---
a/repository/src/test/java/org/apache/atlas/repository/impexp/ImportServiceTest.java
+++
b/repository/src/test/java/org/apache/atlas/repository/impexp/ImportServiceTest.java
@@ -33,6 +33,7 @@ import org.apache.atlas.model.instance.AtlasRelatedObjectId;
import org.apache.atlas.model.instance.AtlasRelationship;
import org.apache.atlas.model.instance.EntityMutationResponse;
import org.apache.atlas.repository.Constants;
+import org.apache.atlas.repository.AtlasTestBase;
import org.apache.atlas.repository.graph.AtlasGraphProvider;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.repository.store.graph.AtlasEntityStore;
@@ -63,8 +64,8 @@ import static
org.apache.atlas.graph.GraphSandboxUtil.useLocalSolr;
import static
org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.getDefaultImportRequest;
import static
org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.getZipSource;
import static
org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.getInputStreamFrom;
-import static
org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.loadModelFromJson;
-import static
org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.loadModelFromResourcesJson;
+import static org.apache.atlas.utils.TestLoadModelUtils.loadModelFromJson;
+import static
org.apache.atlas.utils.TestLoadModelUtils.loadModelFromResourcesJson;
import static
org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.runAndVerifyQuickStart_v1_Import;
import static
org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.runImportWithNoParameters;
import static
org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.runImportWithParameters;
@@ -82,7 +83,7 @@ import static org.testng.Assert.assertNotNull;
import static org.testng.Assert.assertTrue;
@Guice(modules = TestModules.TestOnlyModule.class)
-public class ImportServiceTest extends ExportImportTestBase {
+public class ImportServiceTest extends AtlasTestBase {
private static final int DEFAULT_LIMIT = 25;
private final ImportService importService;
@@ -123,7 +124,7 @@ public class ImportServiceTest extends ExportImportTestBase
{
@AfterTest
public void postTest() throws InterruptedException {
- assertAuditEntry(auditService);
+ assertExportImportAuditEntry(auditService);
}
@DataProvider(name = "sales")
diff --git
a/repository/src/test/java/org/apache/atlas/repository/impexp/ImportTransformsShaperTest.java
b/repository/src/test/java/org/apache/atlas/repository/impexp/ImportTransformsShaperTest.java
index 78fdaca..c2b9dbb 100644
---
a/repository/src/test/java/org/apache/atlas/repository/impexp/ImportTransformsShaperTest.java
+++
b/repository/src/test/java/org/apache/atlas/repository/impexp/ImportTransformsShaperTest.java
@@ -23,6 +23,7 @@ import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.impexp.AtlasImportRequest;
import org.apache.atlas.model.impexp.AtlasImportResult;
import org.apache.atlas.model.instance.AtlasEntity;
+import org.apache.atlas.repository.AtlasTestBase;
import org.apache.atlas.repository.store.graph.AtlasEntityStore;
import org.apache.atlas.store.AtlasTypeDefStore;
import org.apache.atlas.type.AtlasClassificationType;
@@ -35,13 +36,13 @@ import javax.inject.Inject;
import java.io.IOException;
import java.util.List;
-import static
org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.loadFsModel;
+import static org.apache.atlas.utils.TestLoadModelUtils.loadFsModel;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertTrue;
import static org.testng.AssertJUnit.assertNotNull;
@Guice(modules = TestModules.TestOnlyModule.class)
-public class ImportTransformsShaperTest extends ExportImportTestBase {
+public class ImportTransformsShaperTest extends AtlasTestBase {
@Inject
AtlasTypeRegistry typeRegistry;
diff --git
a/repository/src/test/java/org/apache/atlas/repository/impexp/IncrementalExportEntityProviderTest.java
b/repository/src/test/java/org/apache/atlas/repository/impexp/IncrementalExportEntityProviderTest.java
index 42b6353..ed6c12d 100644
---
a/repository/src/test/java/org/apache/atlas/repository/impexp/IncrementalExportEntityProviderTest.java
+++
b/repository/src/test/java/org/apache/atlas/repository/impexp/IncrementalExportEntityProviderTest.java
@@ -22,6 +22,7 @@ package org.apache.atlas.repository.impexp;
import org.apache.atlas.RequestContext;
import org.apache.atlas.TestModules;
import org.apache.atlas.exception.AtlasBaseException;
+import org.apache.atlas.repository.AtlasTestBase;
import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.repository.store.graph.v2.AtlasEntityStoreV2;
import org.apache.atlas.repository.util.UniqueList;
@@ -40,7 +41,7 @@ import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertTrue;
@Guice(modules = TestModules.TestOnlyModule.class)
-public class IncrementalExportEntityProviderTest extends ExportImportTestBase {
+public class IncrementalExportEntityProviderTest extends AtlasTestBase {
@Inject
AtlasTypeRegistry typeRegistry;
diff --git
a/repository/src/test/java/org/apache/atlas/repository/impexp/RelationshipAttributesExtractorTest.java
b/repository/src/test/java/org/apache/atlas/repository/impexp/RelationshipAttributesExtractorTest.java
index a1b512f..5f41cc9 100644
---
a/repository/src/test/java/org/apache/atlas/repository/impexp/RelationshipAttributesExtractorTest.java
+++
b/repository/src/test/java/org/apache/atlas/repository/impexp/RelationshipAttributesExtractorTest.java
@@ -25,6 +25,7 @@ import org.apache.atlas.model.impexp.AtlasExportRequest;
import org.apache.atlas.model.impexp.AtlasExportResult;
import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.model.instance.AtlasObjectId;
+import org.apache.atlas.repository.AtlasTestBase;
import org.apache.atlas.repository.graph.AtlasGraphProvider;
import org.apache.atlas.repository.store.graph.v2.AtlasEntityStoreV2;
import org.apache.atlas.runner.LocalSolrRunner;
@@ -50,7 +51,7 @@ import java.util.HashMap;
import static org.apache.atlas.graph.GraphSandboxUtil.useLocalSolr;
import static
org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.getZipSource;
-import static
org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.loadModelFromJson;
+import static org.apache.atlas.utils.TestLoadModelUtils.loadModelFromJson;
import static
org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.runImportWithNoParameters;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertFalse;
@@ -58,7 +59,7 @@ import static org.testng.Assert.assertNotNull;
import static org.testng.Assert.assertTrue;
@Guice(modules = TestModules.TestOnlyModule.class)
-public class RelationshipAttributesExtractorTest extends ExportImportTestBase {
+public class RelationshipAttributesExtractorTest extends AtlasTestBase {
private static final String EXPORT_FULL = "full";
private static final String EXPORT_CONNECTED = "connected";
diff --git
a/repository/src/test/java/org/apache/atlas/repository/impexp/ReplicationEntityAttributeTest.java
b/repository/src/test/java/org/apache/atlas/repository/impexp/ReplicationEntityAttributeTest.java
index 7a1ed18..ebdc7b5 100644
---
a/repository/src/test/java/org/apache/atlas/repository/impexp/ReplicationEntityAttributeTest.java
+++
b/repository/src/test/java/org/apache/atlas/repository/impexp/ReplicationEntityAttributeTest.java
@@ -31,9 +31,8 @@ import org.apache.atlas.model.impexp.AtlasServer;
import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.model.typedef.AtlasEntityDef;
import org.apache.atlas.repository.Constants;
-import org.apache.atlas.repository.store.graph.v2.AtlasEntityChangeNotifier;
+import org.apache.atlas.repository.AtlasTestBase;
import org.apache.atlas.repository.store.graph.v2.AtlasEntityStoreV2;
-import org.apache.atlas.repository.store.graph.v2.BulkImporterImpl;
import org.apache.atlas.repository.store.graph.v2.EntityGraphMapper;
import org.apache.atlas.store.AtlasTypeDefStore;
import org.apache.atlas.type.AtlasEntityType;
@@ -63,7 +62,7 @@ import static org.testng.Assert.assertNotNull;
import static org.testng.Assert.assertTrue;
@Guice(modules = TestModules.TestOnlyModule.class)
-public class ReplicationEntityAttributeTest extends ExportImportTestBase {
+public class ReplicationEntityAttributeTest extends AtlasTestBase {
private final String ENTITIES_SUB_DIR = "stocksDB-Entities";
private final String EXPORT_REQUEST_FILE = "export-replicatedTo";
private final String IMPORT_REQUEST_FILE = "import-replicatedFrom";
diff --git
a/repository/src/test/java/org/apache/atlas/repository/impexp/StartEntityFetchByExportRequestTest.java
b/repository/src/test/java/org/apache/atlas/repository/impexp/StartEntityFetchByExportRequestTest.java
index 48b9582..c9996b8 100644
---
a/repository/src/test/java/org/apache/atlas/repository/impexp/StartEntityFetchByExportRequestTest.java
+++
b/repository/src/test/java/org/apache/atlas/repository/impexp/StartEntityFetchByExportRequestTest.java
@@ -22,6 +22,7 @@ import org.apache.atlas.TestModules;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.impexp.AtlasExportRequest;
import org.apache.atlas.model.instance.AtlasObjectId;
+import org.apache.atlas.repository.AtlasTestBase;
import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.store.AtlasTypeDefStore;
import org.apache.atlas.type.AtlasType;
@@ -43,7 +44,7 @@ import static
org.apache.atlas.repository.impexp.StartEntityFetchByExportRequest
import static org.testng.Assert.assertEquals;
@Guice(modules = TestModules.TestOnlyModule.class)
-public class StartEntityFetchByExportRequestTest extends ExportImportTestBase {
+public class StartEntityFetchByExportRequestTest extends AtlasTestBase {
@Inject
private AtlasGraph atlasGraph;
diff --git
a/repository/src/test/java/org/apache/atlas/repository/impexp/TableReplicationRequestProcessorTest.java
b/repository/src/test/java/org/apache/atlas/repository/impexp/TableReplicationRequestProcessorTest.java
index c9bb11c..99df914 100644
---
a/repository/src/test/java/org/apache/atlas/repository/impexp/TableReplicationRequestProcessorTest.java
+++
b/repository/src/test/java/org/apache/atlas/repository/impexp/TableReplicationRequestProcessorTest.java
@@ -24,6 +24,7 @@ import org.apache.atlas.TestUtilsV2;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.impexp.AtlasImportRequest;
import org.apache.atlas.model.impexp.ExportImportAuditEntry;
+import org.apache.atlas.repository.AtlasTestBase;
import org.apache.atlas.repository.graph.AtlasGraphProvider;
import org.apache.atlas.repository.store.graph.AtlasEntityStore;
import org.apache.atlas.runner.LocalSolrRunner;
@@ -55,7 +56,7 @@ import static org.testng.Assert.assertNotNull;
@Guice(modules = TestModules.TestOnlyModule.class)
-public class TableReplicationRequestProcessorTest extends ExportImportTestBase
{
+public class TableReplicationRequestProcessorTest extends AtlasTestBase {
private static final Logger LOG =
LoggerFactory.getLogger(TableReplicationRequestProcessorTest.class);
private static final String ENTITY_GUID_REPLICATED =
"718a6d12-35a8-4731-aff8-3a64637a43a3";
diff --git
a/repository/src/test/java/org/apache/atlas/repository/impexp/ZipFileResourceTestUtils.java
b/repository/src/test/java/org/apache/atlas/repository/impexp/ZipFileResourceTestUtils.java
index 0ffc3d5..3f61cf1 100644
---
a/repository/src/test/java/org/apache/atlas/repository/impexp/ZipFileResourceTestUtils.java
+++
b/repository/src/test/java/org/apache/atlas/repository/impexp/ZipFileResourceTestUtils.java
@@ -26,43 +26,25 @@ import org.apache.atlas.model.impexp.AtlasExportResult;
import org.apache.atlas.model.impexp.AtlasImportRequest;
import org.apache.atlas.model.impexp.AtlasImportResult;
import org.apache.atlas.model.instance.AtlasEntity;
-import org.apache.atlas.model.instance.EntityMutationResponse;
-import org.apache.atlas.model.typedef.AtlasEntityDef;
-import org.apache.atlas.model.typedef.AtlasStructDef;
-import org.apache.atlas.model.typedef.AtlasTypesDef;
-import
org.apache.atlas.repository.store.bootstrap.AtlasTypeDefStoreInitializer;
-import org.apache.atlas.repository.store.graph.v2.AtlasEntityStoreV2;
-import org.apache.atlas.repository.store.graph.v2.AtlasEntityStreamForImport;
import org.apache.atlas.repository.store.graph.v2.EntityImportStream;
-import org.apache.atlas.store.AtlasTypeDefStore;
-import org.apache.atlas.type.AtlasType;
-import org.apache.atlas.type.AtlasTypeRegistry;
-import org.apache.atlas.utils.AtlasJson;
import org.apache.atlas.utils.TestResourceFileUtils;
-import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
-import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.testng.SkipException;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
-import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Files;
-import java.util.ArrayList;
-import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import java.util.Objects;
import java.util.Set;
import static org.testng.Assert.assertEquals;
-import static org.testng.Assert.assertFalse;
import static org.testng.Assert.assertNotNull;
import static org.testng.Assert.assertTrue;
@@ -73,87 +55,6 @@ public class ZipFileResourceTestUtils {
return TestResourceFileUtils.getFileInputStream(fileName);
}
- public static List<String> getAllModels(String dirName) throws IOException
{
- List<String> ret = null;
- File topModelsDir = new
File(System.getProperty("user.dir") + "/../addons/models");
- File[] topModelsDirContents = topModelsDir.exists() ?
topModelsDir.listFiles() : null;
-
- assertTrue(topModelsDirContents != null,
topModelsDir.getAbsolutePath() + ": unable to find/read directory");
- if(topModelsDirContents != null) {
- Arrays.sort(topModelsDirContents);
- for (File modelDir : topModelsDirContents) {
- if (modelDir.exists() && modelDir.isDirectory() &&
modelDir.getAbsolutePath().contains(dirName)) {
- File[] models = modelDir.listFiles();
- Arrays.sort(models);
- ret = new ArrayList<>();
- for (File model : Objects.requireNonNull(models)) {
- ret.add(getFileContents(modelDir, model.getName()));
- }
-
- }
-
- if (ret != null && ret.size() > 0) {
- break;
- }
- }
- } else {
- throw new IOException("Unable to retrieve model contents.");
- }
-
- return ret;
- }
-
- public static String getModelJson(String fileName) throws IOException {
- String ret = null;
- File topModelsDir = new File(System.getProperty("user.dir")
+ "/../addons/models");
- File[] topModelsDirContents = topModelsDir.exists() ?
topModelsDir.listFiles() : null;
-
- assertTrue(topModelsDirContents != null,
topModelsDir.getAbsolutePath() + ": unable to find/read directory");
- if(topModelsDirContents != null) {
- Arrays.sort(topModelsDirContents);
- for (File modelDir : topModelsDirContents) {
- if (modelDir.exists() && modelDir.isDirectory()) {
- ret = getFileContents(modelDir, fileName);
-
- if (ret != null) {
- break;
- }
- }
- }
-
- if (ret == null) {
- ret = getFileContents(topModelsDir, fileName);
- }
-
- assertTrue(ret != null, fileName + ": unable to find model file");
- } else {
- throw new IOException("Unable to retrieve model contents.");
- }
-
- return ret;
- }
-
- public static String getFileContents(File dir, String fileName) throws
IOException {
- if (dir.exists() && dir.isDirectory()) {
- File file = new File(dir, fileName);
-
- if (file.exists() && file.isFile()) {
- return FileUtils.readFileToString(file);
- }
- }
-
- return null;
- }
-
- public static String getModelJsonFromResources(String fileName) throws
IOException {
- String filePath = TestResourceFileUtils.getTestFilePath(fileName);
- File f = new File(filePath);
- String s = FileUtils.readFileToString(f);
- assertFalse(StringUtils.isEmpty(s), "Model file read correctly from
resources!");
-
- return s;
- }
-
public static Object[][] getZipSource(String fileName) throws IOException,
AtlasBaseException {
return new Object[][]{{getInputStreamFrom(fileName)}};
}
@@ -184,42 +85,6 @@ public class ZipFileResourceTestUtils {
}
}
- public static AtlasTypesDef loadTypes(String entitiesSubDir, String
fileName) {
- try {
- return TestResourceFileUtils.readObjectFromJson(entitiesSubDir,
fileName, AtlasTypesDef.class);
- } catch (IOException e) {
- throw new SkipException(String.format("createTypes: '%s' could not
be loaded.", fileName));
- }
- }
-
-
- public static AtlasEntity.AtlasEntityWithExtInfo loadEntity(String
entitiesSubDir, String fileName) {
- try {
- return TestResourceFileUtils.readObjectFromJson(entitiesSubDir,
fileName, AtlasEntity.AtlasEntityWithExtInfo.class);
- } catch (IOException e) {
- throw new SkipException(String.format("createTypes: '%s' could not
be loaded.", fileName));
- }
- }
-
- public static void createTypes(AtlasTypeDefStore typeDefStore, String
entitiesSubDir, String typesDef) {
- try {
- typeDefStore.createTypesDef(loadTypes(entitiesSubDir, typesDef));
- } catch (AtlasBaseException e) {
- throw new SkipException("setup: could not load typesDef.");
- }
- }
-
- public static void createAtlasEntity(AtlasEntityStoreV2 entityStoreV1,
AtlasEntity.AtlasEntityWithExtInfo atlasEntity) {
- try {
- EntityMutationResponse response =
entityStoreV1.createOrUpdateForImport(new
AtlasEntityStreamForImport(atlasEntity, null));
- assertNotNull(response);
- assertTrue((response.getCreatedEntities() != null &&
response.getCreatedEntities().size() > 0) ||
- (response.getMutatedEntities() != null &&
response.getMutatedEntities().size() > 0));
- } catch (AtlasBaseException e) {
- throw new SkipException(String.format("createAtlasEntity: could
not loaded '%s'.", atlasEntity.getEntity().getTypeName()));
- }
- }
-
public static InputStream runExportWithParameters(ExportService
exportService, AtlasExportRequest request) {
final String requestingIP = "1.0.0.0";
final String hostName = "localhost";
@@ -262,60 +127,6 @@ public class ZipFileResourceTestUtils {
return entityWithExtInfo;
}
- public static void loadModelFromJson(String fileName, AtlasTypeDefStore
typeDefStore, AtlasTypeRegistry typeRegistry) throws IOException,
AtlasBaseException {
- AtlasTypesDef typesFromJson = getAtlasTypesDefFromFile(fileName);
- addReplicationAttributes(typesFromJson);
- createTypesAsNeeded(typesFromJson, typeDefStore, typeRegistry);
- }
-
- public static void loadAllModels(String dirName, AtlasTypeDefStore
typeDefStore, AtlasTypeRegistry typeRegistry) throws IOException,
AtlasBaseException {
- List<String> allModels = getAllModels(dirName);
- for (String model : allModels) {
- AtlasTypesDef typesFromJson = AtlasJson.fromJson(model,
AtlasTypesDef.class);
- createTypesAsNeeded(typesFromJson, typeDefStore, typeRegistry);
- }
- }
-
- private static void addReplicationAttributes(AtlasTypesDef typesFromJson)
throws IOException {
- if(typesFromJson.getEntityDefs() == null ||
typesFromJson.getEntityDefs().size() == 0) return;
-
- AtlasEntityDef ed = typesFromJson.getEntityDefs().get(0);
- if(!ed.getName().equals("Referenceable")) return;
-
- String replAttr1Json =
TestResourceFileUtils.getJson("stocksDB-Entities","replicationAttrs");
- String replAttr2Json = StringUtils.replace(replAttr1Json, "From",
"To");
-
- ed.addAttribute(AtlasType.fromJson(replAttr1Json,
AtlasStructDef.AtlasAttributeDef.class));
- ed.addAttribute(AtlasType.fromJson(replAttr2Json,
AtlasStructDef.AtlasAttributeDef.class));
- }
-
- public static void loadModelFromResourcesJson(String fileName,
AtlasTypeDefStore typeDefStore, AtlasTypeRegistry typeRegistry) throws
IOException, AtlasBaseException {
- AtlasTypesDef typesFromJson =
getAtlasTypesDefFromResourceFile(fileName);
- createTypesAsNeeded(typesFromJson, typeDefStore, typeRegistry);
- }
-
- public static void createTypesAsNeeded(AtlasTypesDef typesFromJson,
AtlasTypeDefStore typeDefStore, AtlasTypeRegistry typeRegistry) throws
AtlasBaseException {
- if(typesFromJson == null) {
- return;
- }
-
- AtlasTypesDef typesToCreate =
AtlasTypeDefStoreInitializer.getTypesToCreate(typesFromJson, typeRegistry);
- if (typesToCreate != null && !typesToCreate.isEmpty()) {
- typeDefStore.createTypesDef(typesToCreate);
- }
- }
-
- private static AtlasTypesDef getAtlasTypesDefFromFile(String fileName)
throws IOException {
- String sampleTypes = ZipFileResourceTestUtils.getModelJson(fileName);
- if(sampleTypes == null) return null;
- return AtlasType.fromJson(sampleTypes, AtlasTypesDef.class);
- }
-
- private static AtlasTypesDef getAtlasTypesDefFromResourceFile(String
fileName) throws IOException {
- String sampleTypes = getModelJsonFromResources(fileName);
- return AtlasType.fromJson(sampleTypes, AtlasTypesDef.class);
- }
-
public static AtlasImportRequest getDefaultImportRequest() {
return new AtlasImportRequest();
}
@@ -370,16 +181,4 @@ public class ZipFileResourceTestUtils {
verifyImportedMetrics(exportResult, result);
verifyImportedEntities(creationOrder, result.getProcessedEntities());
}
-
- public static void loadBaseModel(AtlasTypeDefStore typeDefStore,
AtlasTypeRegistry typeRegistry) throws IOException, AtlasBaseException {
- loadModelFromJson("0010-base_model.json", typeDefStore, typeRegistry);
- }
-
- public static void loadFsModel(AtlasTypeDefStore typeDefStore,
AtlasTypeRegistry typeRegistry) throws IOException, AtlasBaseException {
- loadModelFromJson("1020-fs_model.json", typeDefStore, typeRegistry);
- }
-
- public static void loadHiveModel(AtlasTypeDefStore typeDefStore,
AtlasTypeRegistry typeRegistry) throws IOException, AtlasBaseException {
- loadModelFromJson("1030-hive_model.json", typeDefStore, typeRegistry);
- }
}
diff --git
a/repository/src/test/java/org/apache/atlas/repository/migration/MigrationBaseAsserts.java
b/repository/src/test/java/org/apache/atlas/repository/migration/MigrationBaseAsserts.java
index 18e950e..ce9056b 100644
---
a/repository/src/test/java/org/apache/atlas/repository/migration/MigrationBaseAsserts.java
+++
b/repository/src/test/java/org/apache/atlas/repository/migration/MigrationBaseAsserts.java
@@ -37,7 +37,7 @@ import java.io.IOException;
import java.util.Iterator;
import static org.apache.atlas.graph.GraphSandboxUtil.useLocalSolr;
-import static
org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.loadModelFromJson;
+import static org.apache.atlas.utils.TestLoadModelUtils.loadModelFromJson;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertNotNull;
import static org.testng.Assert.assertTrue;
diff --git
a/repository/src/test/java/org/apache/atlas/repository/migration/RelationshipCacheGeneratorTest.java
b/repository/src/test/java/org/apache/atlas/repository/migration/RelationshipCacheGeneratorTest.java
index 050bda3..4cd9fc6 100644
---
a/repository/src/test/java/org/apache/atlas/repository/migration/RelationshipCacheGeneratorTest.java
+++
b/repository/src/test/java/org/apache/atlas/repository/migration/RelationshipCacheGeneratorTest.java
@@ -38,7 +38,7 @@ import java.util.Map;
import static
org.apache.atlas.model.typedef.AtlasRelationshipDef.PropagateTags.ONE_TO_TWO;
import static
org.apache.atlas.model.typedef.AtlasRelationshipDef.PropagateTags.TWO_TO_ONE;
-import static
org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.loadModelFromJson;
+import static org.apache.atlas.utils.TestLoadModelUtils.loadModelFromJson;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertTrue;
diff --git
a/repository/src/test/java/org/apache/atlas/repository/store/graph/v2/AtlasNamespaceDefStoreV2Test.java
b/repository/src/test/java/org/apache/atlas/repository/store/graph/v2/AtlasNamespaceDefStoreV2Test.java
index cbac18c..e2f5c16 100644
---
a/repository/src/test/java/org/apache/atlas/repository/store/graph/v2/AtlasNamespaceDefStoreV2Test.java
+++
b/repository/src/test/java/org/apache/atlas/repository/store/graph/v2/AtlasNamespaceDefStoreV2Test.java
@@ -38,7 +38,10 @@ import java.io.IOException;
import java.util.*;
import static
org.apache.atlas.model.typedef.AtlasNamespaceDef.ATTR_OPTION_APPLICABLE_ENTITY_TYPES;
-import static org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.*;
+
+import static org.apache.atlas.utils.TestLoadModelUtils.loadBaseModel;
+import static org.apache.atlas.utils.TestLoadModelUtils.loadFsModel;
+import static org.apache.atlas.utils.TestLoadModelUtils.loadHiveModel;
/* Please note that for these tests, since the typeRegistry can be injected
only once,
* any new tests should make sure that they flush the type registry at the end
of the test.
diff --git
a/repository/src/test/java/org/apache/atlas/repository/tagpropagation/ClassificationPropagationTest.java
b/repository/src/test/java/org/apache/atlas/repository/tagpropagation/ClassificationPropagationTest.java
index 6269d3f..80e20be 100644
---
a/repository/src/test/java/org/apache/atlas/repository/tagpropagation/ClassificationPropagationTest.java
+++
b/repository/src/test/java/org/apache/atlas/repository/tagpropagation/ClassificationPropagationTest.java
@@ -66,7 +66,7 @@ import static
org.apache.atlas.model.typedef.AtlasRelationshipDef.PropagateTags.
import static
org.apache.atlas.model.typedef.AtlasRelationshipDef.PropagateTags.NONE;
import static
org.apache.atlas.model.typedef.AtlasRelationshipDef.PropagateTags.ONE_TO_TWO;
import static
org.apache.atlas.model.typedef.AtlasRelationshipDef.PropagateTags.TWO_TO_ONE;
-import static
org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.loadModelFromJson;
+import static org.apache.atlas.utils.TestLoadModelUtils.loadModelFromJson;
import static
org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.runImportWithNoParameters;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertNotNull;
diff --git
a/repository/src/test/java/org/apache/atlas/repository/userprofile/UserProfileServiceTest.java
b/repository/src/test/java/org/apache/atlas/repository/userprofile/UserProfileServiceTest.java
index e56e483..2410a9c 100644
---
a/repository/src/test/java/org/apache/atlas/repository/userprofile/UserProfileServiceTest.java
+++
b/repository/src/test/java/org/apache/atlas/repository/userprofile/UserProfileServiceTest.java
@@ -44,7 +44,7 @@ import java.util.Optional;
import static org.apache.atlas.graph.GraphSandboxUtil.useLocalSolr;
import static
org.apache.atlas.model.profile.AtlasUserSavedSearch.SavedSearchType.BASIC;
-import static
org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.loadModelFromJson;
+import static org.apache.atlas.utils.TestLoadModelUtils.loadModelFromJson;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertFalse;
import static org.testng.Assert.assertNotNull;
diff --git
a/repository/src/test/java/org/apache/atlas/services/MetricsServiceTest.java
b/repository/src/test/java/org/apache/atlas/services/MetricsServiceTest.java
index baeafd4..4739bca 100644
--- a/repository/src/test/java/org/apache/atlas/services/MetricsServiceTest.java
+++ b/repository/src/test/java/org/apache/atlas/services/MetricsServiceTest.java
@@ -24,7 +24,6 @@ import org.apache.atlas.model.metrics.AtlasMetrics;
import org.apache.atlas.repository.graph.AtlasGraphProvider;
import org.apache.atlas.repository.impexp.ImportService;
import org.apache.atlas.repository.impexp.ZipFileResourceTestUtils;
-import org.apache.atlas.repository.impexp.ZipSource;
import org.apache.atlas.runner.LocalSolrRunner;
import org.apache.atlas.store.AtlasTypeDefStore;
import org.apache.atlas.type.AtlasTypeRegistry;
@@ -37,7 +36,6 @@ import org.testng.annotations.Guice;
import org.testng.annotations.Test;
import javax.inject.Inject;
-import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.time.Clock;
@@ -49,7 +47,7 @@ import java.util.Map;
import static org.apache.atlas.graph.GraphSandboxUtil.useLocalSolr;
import static org.apache.atlas.model.metrics.AtlasMetrics.*;
-import static
org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.loadModelFromJson;
+import static org.apache.atlas.utils.TestLoadModelUtils.loadModelFromJson;
import static
org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.runImportWithNoParameters;
import static org.apache.atlas.services.MetricsService.ENTITY;
import static org.apache.atlas.services.MetricsService.GENERAL;
diff --git
a/repository/src/test/java/org/apache/atlas/repository/impexp/ZipFileResourceTestUtils.java
b/repository/src/test/java/org/apache/atlas/utils/TestLoadModelUtils.java
similarity index 57%
copy from
repository/src/test/java/org/apache/atlas/repository/impexp/ZipFileResourceTestUtils.java
copy to repository/src/test/java/org/apache/atlas/utils/TestLoadModelUtils.java
index 0ffc3d5..f175386 100644
---
a/repository/src/test/java/org/apache/atlas/repository/impexp/ZipFileResourceTestUtils.java
+++ b/repository/src/test/java/org/apache/atlas/utils/TestLoadModelUtils.java
@@ -15,16 +15,10 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.atlas.repository.impexp;
-import com.google.common.collect.Sets;
-import org.apache.atlas.RequestContext;
-import org.apache.atlas.TestUtilsV2;
+package org.apache.atlas.utils;
+
import org.apache.atlas.exception.AtlasBaseException;
-import org.apache.atlas.model.impexp.AtlasExportRequest;
-import org.apache.atlas.model.impexp.AtlasExportResult;
-import org.apache.atlas.model.impexp.AtlasImportRequest;
-import org.apache.atlas.model.impexp.AtlasImportResult;
import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.model.instance.EntityMutationResponse;
import org.apache.atlas.model.typedef.AtlasEntityDef;
@@ -33,45 +27,28 @@ import org.apache.atlas.model.typedef.AtlasTypesDef;
import
org.apache.atlas.repository.store.bootstrap.AtlasTypeDefStoreInitializer;
import org.apache.atlas.repository.store.graph.v2.AtlasEntityStoreV2;
import org.apache.atlas.repository.store.graph.v2.AtlasEntityStreamForImport;
-import org.apache.atlas.repository.store.graph.v2.EntityImportStream;
import org.apache.atlas.store.AtlasTypeDefStore;
import org.apache.atlas.type.AtlasType;
import org.apache.atlas.type.AtlasTypeRegistry;
-import org.apache.atlas.utils.AtlasJson;
-import org.apache.atlas.utils.TestResourceFileUtils;
import org.apache.commons.io.FileUtils;
-import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.testng.SkipException;
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
import java.io.File;
-import java.io.FileInputStream;
import java.io.IOException;
-import java.io.InputStream;
-import java.nio.file.Files;
import java.util.ArrayList;
import java.util.Arrays;
-import java.util.HashMap;
import java.util.List;
-import java.util.Map;
import java.util.Objects;
-import java.util.Set;
-import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertFalse;
import static org.testng.Assert.assertNotNull;
import static org.testng.Assert.assertTrue;
-public class ZipFileResourceTestUtils {
- public static final Logger LOG =
LoggerFactory.getLogger(ZipFileResourceTestUtils.class);
-
- public static FileInputStream getFileInputStream(String fileName) {
- return TestResourceFileUtils.getFileInputStream(fileName);
- }
+public class TestLoadModelUtils {
+ public static final Logger LOG =
LoggerFactory.getLogger(TestLoadModelUtils.class);
public static List<String> getAllModels(String dirName) throws IOException
{
List<String> ret = null;
@@ -154,41 +131,11 @@ public class ZipFileResourceTestUtils {
return s;
}
- public static Object[][] getZipSource(String fileName) throws IOException,
AtlasBaseException {
- return new Object[][]{{getInputStreamFrom(fileName)}};
- }
-
- public static InputStream getInputStreamFrom(String fileName) {
- return ZipFileResourceTestUtils.getFileInputStream(fileName);
- }
-
- public static void verifyImportedEntities(List<String> creationOrder,
List<String> processedEntities) {
- Set<String> lhs =
com.google.common.collect.Sets.newHashSet(creationOrder);
- Set<String> rhs =
com.google.common.collect.Sets.newHashSet(processedEntities);
- Set<String> difference = Sets.difference(lhs, rhs);
-
- assertNotNull(difference);
- assertEquals(difference.size(), 0);
- }
-
- public static void verifyImportedMetrics(AtlasExportResult exportResult,
AtlasImportResult importResult) {
- Map<String, Integer> metricsForCompare =
getImportMetricsForCompare(importResult);
- for (Map.Entry<String, Integer> entry :
exportResult.getMetrics().entrySet()) {
- if(entry.getKey().startsWith("entity") == false ||
- entry.getKey().contains("withExtInfo") ||
- entry.getKey().contains("Column") ||
- entry.getKey().contains("StorageDesc")) continue;
-
- assertTrue(metricsForCompare.containsKey(entry.getKey()),
entry.getKey());
- assertEquals(entry.getValue(),
metricsForCompare.get(entry.getKey()), entry.getKey());
- }
- }
-
public static AtlasTypesDef loadTypes(String entitiesSubDir, String
fileName) {
try {
return TestResourceFileUtils.readObjectFromJson(entitiesSubDir,
fileName, AtlasTypesDef.class);
} catch (IOException e) {
- throw new SkipException(String.format("createTypes: '%s' could not
be loaded.", fileName));
+ throw new SkipException(String.format("loadTypes: '%s' could not
be loaded.", fileName));
}
}
@@ -197,7 +144,7 @@ public class ZipFileResourceTestUtils {
try {
return TestResourceFileUtils.readObjectFromJson(entitiesSubDir,
fileName, AtlasEntity.AtlasEntityWithExtInfo.class);
} catch (IOException e) {
- throw new SkipException(String.format("createTypes: '%s' could not
be loaded.", fileName));
+ throw new SkipException(String.format("loadEntity: '%s' could not
be loaded.", fileName));
}
}
@@ -220,48 +167,6 @@ public class ZipFileResourceTestUtils {
}
}
- public static InputStream runExportWithParameters(ExportService
exportService, AtlasExportRequest request) {
- final String requestingIP = "1.0.0.0";
- final String hostName = "localhost";
- final String userName = "admin";
-
- try {
- ByteArrayOutputStream baos = new ByteArrayOutputStream();
- ZipSink zipSink = new ZipSink(baos);
-
- AtlasExportResult result = exportService.run(zipSink, request,
userName, hostName, requestingIP);
- assertEquals(result.getOperationStatus(),
AtlasExportResult.OperationStatus.SUCCESS);
-
- zipSink.close();
- return new ByteArrayInputStream(baos.toByteArray());
- }
- catch(Exception ex) {
- throw new SkipException(String.format("runExportWithParameters:
%s: failed!", request.toString()));
- }
- }
-
- private static Map<String,Integer>
getImportMetricsForCompare(AtlasImportResult result) {
- Map<String, Integer> r = new HashMap<>();
- for (Map.Entry<String, Integer> entry :
result.getMetrics().entrySet()) {
- r.put(entry.getKey().replace(":updated", "").replace(":created",
""), entry.getValue());
- }
-
- return r;
- }
-
- public static AtlasEntity.AtlasEntityWithExtInfo getEntities(ZipSource
source, int expectedCount) {
- AtlasEntity.AtlasEntityWithExtInfo entityWithExtInfo = new
AtlasEntity.AtlasEntityWithExtInfo();
- int count = 0;
- for (String s : source.getCreationOrder()) {
- AtlasEntity entity = source.getByGuid(s);
- entityWithExtInfo.addReferredEntity(s, entity);
- count++;
- }
-
- assertEquals(count, expectedCount);
- return entityWithExtInfo;
- }
-
public static void loadModelFromJson(String fileName, AtlasTypeDefStore
typeDefStore, AtlasTypeRegistry typeRegistry) throws IOException,
AtlasBaseException {
AtlasTypesDef typesFromJson = getAtlasTypesDefFromFile(fileName);
addReplicationAttributes(typesFromJson);
@@ -306,7 +211,7 @@ public class ZipFileResourceTestUtils {
}
private static AtlasTypesDef getAtlasTypesDefFromFile(String fileName)
throws IOException {
- String sampleTypes = ZipFileResourceTestUtils.getModelJson(fileName);
+ String sampleTypes = getModelJson(fileName);
if(sampleTypes == null) return null;
return AtlasType.fromJson(sampleTypes, AtlasTypesDef.class);
}
@@ -316,61 +221,6 @@ public class ZipFileResourceTestUtils {
return AtlasType.fromJson(sampleTypes, AtlasTypesDef.class);
}
- public static AtlasImportRequest getDefaultImportRequest() {
- return new AtlasImportRequest();
- }
-
-
- public static AtlasImportResult runImportWithParameters(ImportService
importService, AtlasImportRequest request, InputStream inputStream) throws
AtlasBaseException, IOException {
- final String requestingIP = "1.0.0.0";
- final String hostName = "localhost";
- final String userName = "admin";
-
- AtlasImportResult result = importService.run(inputStream, request,
userName, hostName, requestingIP);
- assertEquals(result.getOperationStatus(),
AtlasImportResult.OperationStatus.SUCCESS);
- return result;
- }
-
- public static AtlasImportResult runImportWithNoParameters(ImportService
importService, InputStream inputStream) throws AtlasBaseException, IOException {
- final String requestingIP = "1.0.0.0";
- final String hostName = "localhost";
- final String userName = "admin";
-
- AtlasImportResult result = importService.run(inputStream, userName,
hostName, requestingIP);
- assertEquals(result.getOperationStatus(),
AtlasImportResult.OperationStatus.SUCCESS);
- return result;
- }
-
- public static AtlasImportResult
runImportWithNoParametersUsingBackingDirectory(ImportService importService,
InputStream inputStream) throws AtlasBaseException, IOException {
- final String requestingIP = "1.0.0.0";
- final String hostName = "localhost";
- final String userName = "admin";
-
- EntityImportStream sourceWithBackingDirectory = new
ZipSourceWithBackingDirectory(inputStream,
Files.createTempDirectory("temp").toString());
- AtlasImportResult result =
importService.run(sourceWithBackingDirectory, new AtlasImportRequest(),
userName, hostName, requestingIP);
- assertEquals(result.getOperationStatus(),
AtlasImportResult.OperationStatus.SUCCESS);
- return result;
- }
-
- public static void runAndVerifyQuickStart_v1_Import(ImportService
importService, InputStream is) throws AtlasBaseException, IOException {
- final ByteArrayOutputStream baos = new ByteArrayOutputStream();
- IOUtils.copy(is, baos);
-
- ZipSource zipSource = new ZipSource(new
ByteArrayInputStream(baos.toByteArray()));
- AtlasExportResult exportResult = zipSource.getExportResult();
- List<String> creationOrder = zipSource.getCreationOrder();
-
- RequestContext.clear();
- RequestContext.get().setUser(TestUtilsV2.TEST_USER, null);
-
- AtlasImportRequest request = getDefaultImportRequest();
- AtlasImportResult result = runImportWithParameters(importService,
request, new ByteArrayInputStream(baos.toByteArray()));
-
- assertNotNull(result);
- verifyImportedMetrics(exportResult, result);
- verifyImportedEntities(creationOrder, result.getProcessedEntities());
- }
-
public static void loadBaseModel(AtlasTypeDefStore typeDefStore,
AtlasTypeRegistry typeRegistry) throws IOException, AtlasBaseException {
loadModelFromJson("0010-base_model.json", typeDefStore, typeRegistry);
}
diff --git
a/repository/src/test/resources/json/auditSearchParameters/audit-search-parameter-purge.json
b/repository/src/test/resources/json/auditSearchParameters/audit-search-parameter-purge.json
new file mode 100644
index 0000000..005df6d
--- /dev/null
+++
b/repository/src/test/resources/json/auditSearchParameters/audit-search-parameter-purge.json
@@ -0,0 +1,14 @@
+{
+ "auditFilters": {
+ "condition": "AND",
+ "criterion": [
+ {
+ "attributeName": "operation",
+ "operator": "eq",
+ "attributeValue": "PURGE"
+ }
+ ]
+ },
+ "limit": 10,
+ "offset": 0
+}
\ No newline at end of file
diff --git
a/repository/src/test/resources/json/auditSearchParameters/audit-search-parameter-without-filter.json
b/repository/src/test/resources/json/auditSearchParameters/audit-search-parameter-without-filter.json
new file mode 100644
index 0000000..d74df46
--- /dev/null
+++
b/repository/src/test/resources/json/auditSearchParameters/audit-search-parameter-without-filter.json
@@ -0,0 +1,5 @@
+{
+ "auditFilters": null,
+ "limit": 10,
+ "offset": 0
+}
\ No newline at end of file
diff --git a/server-api/src/main/java/org/apache/atlas/RequestContext.java
b/server-api/src/main/java/org/apache/atlas/RequestContext.java
index f9ca7a2..282a66f 100644
--- a/server-api/src/main/java/org/apache/atlas/RequestContext.java
+++ b/server-api/src/main/java/org/apache/atlas/RequestContext.java
@@ -51,7 +51,6 @@ public class RequestContext {
private final long requestTime
= System.currentTimeMillis();
private final Map<String, AtlasEntityHeader> updatedEntities
= new HashMap<>();
private final Map<String, AtlasEntityHeader> deletedEntities
= new HashMap<>();
- private final Map<String, AtlasEntityHeader> purgedEntities =
new HashMap<>();
private final Map<String, AtlasEntity> entityCache
= new HashMap<>();
private final Map<String, AtlasEntityWithExtInfo> entityExtInfoCache
= new HashMap<>();
private final Map<String, List<AtlasClassification>> addedPropagations
= new HashMap<>();
@@ -110,7 +109,6 @@ public class RequestContext {
public void clearCache() {
this.updatedEntities.clear();
this.deletedEntities.clear();
- this.purgedEntities.clear();
this.entityCache.clear();
this.entityExtInfoCache.clear();
this.addedPropagations.clear();
@@ -228,12 +226,6 @@ public class RequestContext {
}
}
- public void recordEntityPurge(AtlasEntityHeader entity) {
- if (entity != null && entity.getGuid() != null) {
- purgedEntities.put(entity.getGuid(), entity);
- }
- }
-
public void recordAddedPropagation(String guid, AtlasClassification
classification) {
if (StringUtils.isNotEmpty(guid) && classification != null) {
List<AtlasClassification> classifications =
addedPropagations.get(guid);
@@ -314,10 +306,6 @@ public class RequestContext {
return deletedEntities.values();
}
- public Collection<AtlasEntityHeader> getPurgedEntities() {
- return purgedEntities.values();
- }
-
/**
* Checks if an instance with the given guid is in the cache for this
request. Either returns the instance
* or null if it is not in the cache.
@@ -345,10 +333,6 @@ public class RequestContext {
return deletedEntities.containsKey(guid);
}
- public boolean isPurgedEntity(String guid) {
- return purgedEntities.containsKey(guid);
- }
-
public MetricRecorder startMetricRecord(String name) { return metrics !=
null ? metrics.getMetricRecorder(name) : null; }
public void endMetricRecord(MetricRecorder recorder) {
diff --git
a/webapp/src/main/java/org/apache/atlas/web/resources/AdminResource.java
b/webapp/src/main/java/org/apache/atlas/web/resources/AdminResource.java
index 40fd6ef..485d977 100755
--- a/webapp/src/main/java/org/apache/atlas/web/resources/AdminResource.java
+++ b/webapp/src/main/java/org/apache/atlas/web/resources/AdminResource.java
@@ -22,12 +22,16 @@ import com.sun.jersey.multipart.FormDataParam;
import org.apache.atlas.ApplicationProperties;
import org.apache.atlas.AtlasClient;
import org.apache.atlas.AtlasErrorCode;
+import org.apache.atlas.RequestContext;
import org.apache.atlas.authorize.AtlasAdminAccessRequest;
import org.apache.atlas.authorize.AtlasAuthorizationUtils;
import org.apache.atlas.authorize.AtlasEntityAccessRequest;
import org.apache.atlas.authorize.AtlasPrivilege;
import org.apache.atlas.discovery.SearchContext;
import org.apache.atlas.exception.AtlasBaseException;
+import org.apache.atlas.model.audit.AtlasAuditEntry;
+import org.apache.atlas.model.audit.AtlasAuditEntry.AuditOperation;
+import org.apache.atlas.model.audit.AuditSearchParameters;
import org.apache.atlas.model.impexp.AtlasExportRequest;
import org.apache.atlas.model.impexp.AtlasExportResult;
import org.apache.atlas.model.impexp.AtlasImportRequest;
@@ -37,9 +41,11 @@ import org.apache.atlas.model.impexp.ExportImportAuditEntry;
import org.apache.atlas.model.impexp.MigrationStatus;
import org.apache.atlas.model.instance.AtlasCheckStateRequest;
import org.apache.atlas.model.instance.AtlasCheckStateResult;
+import org.apache.atlas.model.instance.AtlasEntityHeader;
import org.apache.atlas.model.instance.EntityMutationResponse;
import org.apache.atlas.model.metrics.AtlasMetrics;
import org.apache.atlas.model.patches.AtlasPatch.AtlasPatches;
+import org.apache.atlas.repository.audit.AtlasAuditService;
import org.apache.atlas.repository.impexp.AtlasServerService;
import org.apache.atlas.repository.impexp.ExportImportAuditService;
import org.apache.atlas.repository.impexp.ExportService;
@@ -90,6 +96,7 @@ import java.io.IOException;
import java.io.InputStream;
import java.util.Arrays;
import java.util.Collection;
+import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
@@ -140,6 +147,7 @@ public class AdminResource {
private final AtlasServerService atlasServerService;
private final AtlasEntityStore entityStore;
private final AtlasPatchManager patchManager;
+ private final AtlasAuditService auditService;
static {
try {
@@ -155,7 +163,7 @@ public class AdminResource {
MigrationProgressService migrationProgressService,
AtlasServerService serverService,
ExportImportAuditService exportImportAuditService,
AtlasEntityStore entityStore,
- AtlasPatchManager patchManager) {
+ AtlasPatchManager patchManager, AtlasAuditService
auditService) {
this.serviceState = serviceState;
this.metricsService = metricsService;
this.exportService = exportService;
@@ -168,6 +176,7 @@ public class AdminResource {
this.exportImportAuditService = exportImportAuditService;
this.importExportOperationLock = new ReentrantLock();
this.patchManager = patchManager;
+ this.auditService = auditService;
}
/**
@@ -448,10 +457,23 @@ public class AdminResource {
try {
if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) {
- perf = AtlasPerfTracer.getPerfTracer(PERF_LOG,
"AdminResource.purgeByGuids(" + guids + ")");
+ perf = AtlasPerfTracer.getPerfTracer(PERF_LOG,
"AdminResource.purgeByIds(" + guids + ")");
}
- return entityStore.purgeByIds(guids);
+ EntityMutationResponse resp = entityStore.purgeByIds(guids);
+
+ final List<AtlasEntityHeader> purgedEntities =
resp.getPurgedEntities();
+ if(purgedEntities != null && purgedEntities.size() > 0) {
+ final String clientId =
RequestContext.get().getClientIPAddress();
+ final Date startTime = new
Date(RequestContext.get().getRequestTime());
+ final Date endTime = new Date();
+
+ auditService.add(AtlasAuthorizationUtils.getCurrentUserName(),
AuditOperation.PURGE,
+ clientId != null ? clientId : "", startTime, endTime,
guids.toString(),
+ resp.getPurgedEntitiesIds());
+ }
+
+ return resp;
} finally {
AtlasPerfTracer.log(perf);
}
@@ -548,6 +570,24 @@ public class AdminResource {
}
}
+ @POST
+ @Path("/audits")
+ @Consumes(Servlets.JSON_MEDIA_TYPE)
+ @Produces(Servlets.JSON_MEDIA_TYPE)
+ public List<AtlasAuditEntry> getAtlasAudits(AuditSearchParameters
auditSearchParameters) throws AtlasBaseException {
+ AtlasPerfTracer perf = null;
+
+ try {
+ if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) {
+ perf = AtlasPerfTracer.getPerfTracer(PERF_LOG,
"getAtlasAudit(" + auditSearchParameters + ")");
+ }
+
+ return auditService.get(auditSearchParameters);
+ } finally {
+ AtlasPerfTracer.log(perf);
+ }
+ }
+
@GET
@Path("activeSearches")
@Produces(Servlets.JSON_MEDIA_TYPE)
diff --git
a/webapp/src/test/java/org/apache/atlas/web/adapters/TestEntitiesREST.java
b/webapp/src/test/java/org/apache/atlas/web/adapters/TestEntitiesREST.java
index 65a9aa3..615bc0f 100644
--- a/webapp/src/test/java/org/apache/atlas/web/adapters/TestEntitiesREST.java
+++ b/webapp/src/test/java/org/apache/atlas/web/adapters/TestEntitiesREST.java
@@ -31,7 +31,7 @@ import static
org.apache.atlas.repository.Constants.MODIFICATION_TIMESTAMP_PROPE
import static org.apache.atlas.repository.Constants.STATE_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.TIMESTAMP_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.TYPE_NAME_PROPERTY_KEY;
-import static
org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.createTypesAsNeeded;
+import static org.apache.atlas.utils.TestLoadModelUtils.createTypesAsNeeded;
import org.apache.atlas.AtlasClient;
import org.apache.atlas.RequestContext;
diff --git
a/webapp/src/test/java/org/apache/atlas/web/integration/EntityV2JerseyResourceIT.java
b/webapp/src/test/java/org/apache/atlas/web/integration/EntityV2JerseyResourceIT.java
index a4b7b0d..cc883d6 100755
---
a/webapp/src/test/java/org/apache/atlas/web/integration/EntityV2JerseyResourceIT.java
+++
b/webapp/src/test/java/org/apache/atlas/web/integration/EntityV2JerseyResourceIT.java
@@ -26,18 +26,16 @@ import org.apache.atlas.AtlasClient;
import org.apache.atlas.AtlasServiceException;
import org.apache.atlas.EntityAuditEvent;
import org.apache.atlas.model.TimeBoundary;
-import org.apache.atlas.model.instance.AtlasClassification;
+import org.apache.atlas.model.audit.AtlasAuditEntry;
+import org.apache.atlas.model.audit.AuditSearchParameters;
+import org.apache.atlas.model.instance.*;
import
org.apache.atlas.model.instance.AtlasClassification.AtlasClassifications;
-import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.model.instance.AtlasEntity.AtlasEntityWithExtInfo;
-import org.apache.atlas.model.instance.AtlasEntityHeader;
-import org.apache.atlas.model.instance.AtlasObjectId;
-import org.apache.atlas.model.instance.EntityMutationResponse;
-import org.apache.atlas.model.instance.EntityMutations;
import org.apache.atlas.model.typedef.AtlasClassificationDef;
import org.apache.atlas.model.typedef.AtlasEntityDef;
import org.apache.atlas.model.typedef.AtlasTypesDef;
import org.apache.atlas.type.AtlasTypeUtil;
+import org.apache.atlas.utils.TestResourceFileUtils;
import org.apache.atlas.v1.typesystem.types.utils.TypesUtil;
import org.apache.commons.lang.RandomStringUtils;
import org.apache.hadoop.util.StringUtils;
@@ -55,7 +53,6 @@ import java.util.stream.Stream;
import static org.testng.Assert.*;
-
/**
* Integration tests for Entity Jersey Resource.
*/
@@ -790,7 +787,8 @@ public class EntityV2JerseyResourceIT extends
BaseResourceIT {
assertNotNull(deleteResponse.getEntitiesByOperation(EntityMutations.EntityOperation.DELETE));
assertEquals(deleteResponse.getEntitiesByOperation(EntityMutations.EntityOperation.DELETE).size(),
2);
- Thread.sleep(1000);
+ //Wait for delete operation
+ Thread.sleep(5000);
// Purge the database entities
Set<String> guids = Stream.of(entity1Header.getGuid(),
entity2Header.getGuid()).collect(Collectors.toSet());
@@ -800,6 +798,12 @@ public class EntityV2JerseyResourceIT extends
BaseResourceIT {
assertNotNull(purgeResponse);
assertNotNull(purgeResponse.getEntitiesByOperation(EntityMutations.EntityOperation.PURGE));
assertEquals(purgeResponse.getEntitiesByOperation(EntityMutations.EntityOperation.PURGE).size(),
2);
+
+ AuditSearchParameters auditSearchParameters =
TestResourceFileUtils.readObjectFromJson("audit-search-parameter-purge",
+ AuditSearchParameters.class);
+ List<AtlasAuditEntry> res =
atlasClientV2.getAtlasAuditByOperation(auditSearchParameters);
+ // Verify that the audit entry is set
+ assertNotNull(res);
}
@Test
diff --git
a/webapp/src/test/java/org/apache/atlas/web/resources/AdminResourceTest.java
b/webapp/src/test/java/org/apache/atlas/web/resources/AdminResourceTest.java
index 563a16f..43b85ef 100644
--- a/webapp/src/test/java/org/apache/atlas/web/resources/AdminResourceTest.java
+++ b/webapp/src/test/java/org/apache/atlas/web/resources/AdminResourceTest.java
@@ -51,7 +51,7 @@ public class AdminResourceTest {
when(serviceState.getState()).thenReturn(ServiceState.ServiceStateValue.ACTIVE);
- AdminResource adminResource = new AdminResource(serviceState, null,
null, null, null, null, null, null, null, null, null);
+ AdminResource adminResource = new AdminResource(serviceState, null,
null, null, null, null, null, null, null, null, null, null);
Response response = adminResource.getStatus();
assertEquals(response.getStatus(), HttpServletResponse.SC_OK);
JsonNode entity = AtlasJson.parseToV1JsonNode((String)
response.getEntity());
@@ -62,7 +62,7 @@ public class AdminResourceTest {
public void testResourceGetsValueFromServiceState() throws IOException {
when(serviceState.getState()).thenReturn(ServiceState.ServiceStateValue.PASSIVE);
- AdminResource adminResource = new AdminResource(serviceState, null,
null, null, null, null, null, null, null, null, null);
+ AdminResource adminResource = new AdminResource(serviceState, null,
null, null, null, null, null, null, null, null, null, null);
Response response = adminResource.getStatus();
verify(serviceState).getState();
diff --git a/webapp/src/test/resources/json/audit-search-parameter-purge.json
b/webapp/src/test/resources/json/audit-search-parameter-purge.json
new file mode 100644
index 0000000..af200ef
--- /dev/null
+++ b/webapp/src/test/resources/json/audit-search-parameter-purge.json
@@ -0,0 +1,24 @@
+{
+ "auditFilters": {
+ "condition": "AND",
+ "criterion": [
+ {
+ "attributeName": "operation",
+ "operator": "like",
+ "attributeValue": "PURGE"
+ },
+ {
+ "attributeName": "startTime",
+ "operator": "gte",
+ "attributeValue": "1575958152162"
+ },
+ {
+ "attributeName": "endTime",
+ "operator": "gte",
+ "attributeValue": "1575958152184"
+ }
+ ]
+ },
+ "limit": 10,
+ "offset": 0
+}
\ No newline at end of file
