[2/8] atlas git commit: ATLAS-2845: Rename AtlasCluster to AtlasServer.
ATLAS-2845: Rename AtlasCluster to AtlasServer.
Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/116fb62c
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/116fb62c
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/116fb62c
Branch: refs/heads/master
Commit: 116fb62c9a26efeff389d280f26e3c172ef98cf8
Parents: c0a91c7
Author: Ashutosh Mestry
Authored: Wed Aug 29 21:20:19 2018 -0700
Committer: Ashutosh Mestry
Committed: Thu Oct 11 14:05:48 2018 -0700
--
addons/models/-Area0/0010-base_model.json | 16 +-
.../001-base_model_replication_attributes.json | 8 +-
.../java/org/apache/atlas/AtlasBaseClient.java | 10 +-
.../org/apache/atlas/repository/Constants.java | 4 +-
.../apache/atlas/model/impexp/AtlasCluster.java | 154 -
.../apache/atlas/model/impexp/AtlasServer.java | 168 +++
.../model/impexp/ExportImportAuditEntry.java| 8 +-
.../repository/impexp/AtlasServerService.java | 165 ++
.../atlas/repository/impexp/AuditsWriter.java | 83 -
.../atlas/repository/impexp/ClusterService.java | 143
.../impexp/ExportImportAuditService.java| 14 +-
.../atlas/repository/ogm/AtlasClusterDTO.java | 82 -
.../atlas/repository/ogm/AtlasServerDTO.java| 85 ++
.../ogm/ExportImportAuditEntryDTO.java | 14 +-
.../test/java/org/apache/atlas/TestModules.java | 4 +-
.../impexp/AtlasServerServiceTest.java | 121 +
.../repository/impexp/ClusterServiceTest.java | 121 -
.../impexp/ExportImportAuditServiceTest.java| 14 +-
.../impexp/ReplicationEntityAttributeTest.java | 19 +--
.../stocksDB-Entities/replicationAttrs.json | 4 +-
.../atlas/web/resources/AdminResource.java | 37 ++--
.../web/resources/AdminExportImportTestIT.java | 18 +-
22 files changed, 671 insertions(+), 621 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/atlas/blob/116fb62c/addons/models/-Area0/0010-base_model.json
--
diff --git a/addons/models/-Area0/0010-base_model.json
b/addons/models/-Area0/0010-base_model.json
index 4e2..3af0e0e 100644
--- a/addons/models/-Area0/0010-base_model.json
+++ b/addons/models/-Area0/0010-base_model.json
@@ -100,12 +100,20 @@
]
},
{
- "name": "AtlasCluster",
+ "name": "AtlasServer",
"typeVersion": "1.0",
"superTypes": [
],
"attributeDefs": [
{
+ "name": "name",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": true,
+ "isOptional": false,
+ "isUnique": false
+},
+{
"name": "displayName",
"typeName": "string",
"cardinality": "SINGLE",
@@ -114,7 +122,7 @@
"isUnique": false
},
{
- "name": "qualifiedName",
+ "name": "fullName",
"typeName": "string",
"cardinality": "SINGLE",
"isIndexable": true,
@@ -258,7 +266,7 @@
"isUnique": false
},
{
- "name": "sourceClusterName",
+ "name": "sourceServerName",
"typeName": "string",
"cardinality": "SINGLE",
"isIndexable": true,
@@ -266,7 +274,7 @@
"isUnique": false
},
{
- "name": "targetClusterName",
+ "name": "targetServerName",
"typeName": "string",
"cardinality": "SINGLE",
"isIndexable": true,
http://git-wip-us.apache.org/repos/asf/atlas/blob/116fb62c/addons/models/-Area0/patches/001-base_model_replication_attributes.json
--
diff --git
a/addons/models/-Area0/patches/001-base_model_replication_attributes.json
b/addons/models/-Area0/patches/001-base_model_replication_attributes.json
index bee3718..323a818 100644
---
a/addons/models/-Area0/patches/001-base_model_replication_attributes.json
+++
b/addons/models/-Area0/patches/001-base_model_replication_attributes.json
@@ -8,16 +8,16 @@
"params": null,
"attributeDefs": [
{
- "name": "replicatedFromCluster",
- "typeName": "array",
+ "name": "replicatedFrom",
+ "typeName": "array",
"cardinality": "SET",
"isIndexable": false,
"isOptional": true,
"isUnique": false
},
{
- "name": "replicatedToCluster",
- "typeName":
[1/8] atlas git commit: ATLAS-2845: Rename AtlasCluster to AtlasServer.
Repository: atlas
Updated Branches:
refs/heads/master c0a91c7e1 -> 8746b3063
http://git-wip-us.apache.org/repos/asf/atlas/blob/116fb62c/webapp/src/test/java/org/apache/atlas/web/resources/AdminExportImportTestIT.java
--
diff --git
a/webapp/src/test/java/org/apache/atlas/web/resources/AdminExportImportTestIT.java
b/webapp/src/test/java/org/apache/atlas/web/resources/AdminExportImportTestIT.java
index d156054..babf275 100644
---
a/webapp/src/test/java/org/apache/atlas/web/resources/AdminExportImportTestIT.java
+++
b/webapp/src/test/java/org/apache/atlas/web/resources/AdminExportImportTestIT.java
@@ -21,7 +21,7 @@ package org.apache.atlas.web.resources;
import org.apache.atlas.AtlasServiceException;
import org.apache.atlas.exception.AtlasBaseException;
-import org.apache.atlas.model.impexp.AtlasCluster;
+import org.apache.atlas.model.impexp.AtlasServer;
import org.apache.atlas.model.impexp.AtlasExportRequest;
import org.apache.atlas.model.impexp.AtlasImportRequest;
import org.apache.atlas.model.impexp.AtlasImportResult;
@@ -44,10 +44,10 @@ import static org.testng.Assert.assertTrue;
public class AdminExportImportTestIT extends BaseResourceIT {
private final String FILE_TO_IMPORT = "stocks-base.zip";
private final String EXPORT_REQUEST_FILE = "export-incremental";
-private final String SOURCE_CLUSTER_NAME = "cl1";
+private final String SOURCE_SERVER_NAME = "cl1";
static final String IMPORT_TRANSFORM_CLEAR_ATTRS =
-"{ \"Asset\": { \"*\":[
\"clearAttrValue:replicatedToCluster,replicatedFromCluster\" ] } }";
+"{ \"Asset\": { \"*\":[
\"clearAttrValue:replicatedTo,replicatedFrom\" ] } }";
static final String IMPORT_TRANSFORM_SET_DELETED =
"{ \"Asset\": { \"*\":[ \"setDeleted\" ] } }";
@@ -77,7 +77,7 @@ public class AdminExportImportTestIT extends BaseResourceIT {
private void performImport(String fileToImport) throws
AtlasServiceException {
AtlasImportRequest request = new AtlasImportRequest();
-
request.getOptions().put(AtlasImportRequest.OPTION_KEY_REPLICATED_FROM,
SOURCE_CLUSTER_NAME);
+
request.getOptions().put(AtlasImportRequest.OPTION_KEY_REPLICATED_FROM,
SOURCE_SERVER_NAME);
request.getOptions().put(AtlasImportRequest.TRANSFORMS_KEY,
IMPORT_TRANSFORM_CLEAR_ATTRS);
performImport(fileToImport, request);
@@ -100,11 +100,11 @@ public class AdminExportImportTestIT extends
BaseResourceIT {
assertEquals(result.getProcessedEntities().size(), 37);
}
-private void assertReplicationData(String clusterName) throws
AtlasServiceException {
-AtlasCluster cluster = atlasClientV2.getCluster(clusterName);
-assertNotNull(cluster);
-assertNotNull(cluster.getAdditionalInfo());
-assertTrue(cluster.getAdditionalInfo().size() > 0);
+private void assertReplicationData(String serverName) throws
AtlasServiceException {
+AtlasServer server = atlasClientV2.getServer(serverName);
+assertNotNull(server);
+assertNotNull(server.getAdditionalInfo());
+assertTrue(server.getAdditionalInfo().size() > 0);
}
@AfterClass
[7/8] atlas git commit: ATLAS-2864: Improved incremental export queries.
ATLAS-2864: Improved incremental export queries.
Signed-off-by: Ashutosh Mestry
Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/dc6be8e2
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/dc6be8e2
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/dc6be8e2
Branch: refs/heads/master
Commit: dc6be8e29353d454732f571a0d410b7871eaff4b
Parents: c646600
Author: Ashutosh Mestry
Authored: Wed Sep 12 21:51:09 2018 -0700
Committer: Ashutosh Mestry
Committed: Thu Oct 11 15:14:17 2018 -0700
--
.../atlas/model/impexp/AtlasExportRequest.java | 33 ++--
.../atlas/repository/impexp/ExportService.java | 186 +++
.../impexp/HdfsPathEntityCreator.java | 8 +-
.../impexp/IncrementalExportEntityProvider.java | 111 +++
.../atlas/repository/util/UniqueList.java | 6 +
.../IncrementalExportEntityProviderTest.java| 94 ++
.../stocksDB-Entities/export-incremental.json | 1 +
7 files changed, 350 insertions(+), 89 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/atlas/blob/dc6be8e2/intg/src/main/java/org/apache/atlas/model/impexp/AtlasExportRequest.java
--
diff --git
a/intg/src/main/java/org/apache/atlas/model/impexp/AtlasExportRequest.java
b/intg/src/main/java/org/apache/atlas/model/impexp/AtlasExportRequest.java
index 7bb8b76..e78bb53 100644
--- a/intg/src/main/java/org/apache/atlas/model/impexp/AtlasExportRequest.java
+++ b/intg/src/main/java/org/apache/atlas/model/impexp/AtlasExportRequest.java
@@ -80,18 +80,6 @@ public class AtlasExportRequest implements Serializable {
this.options = options;
}
-public String getMatchTypeOptionValue() {
-String matchType = null;
-
-if (MapUtils.isNotEmpty(getOptions())) {
-if (getOptions().get(OPTION_ATTR_MATCH_TYPE) != null) {
-matchType =
getOptions().get(OPTION_ATTR_MATCH_TYPE).toString();
-}
-}
-
-return matchType;
-}
-
public String getFetchTypeOptionValue() {
if(getOptions() == null ||
!getOptions().containsKey(OPTION_FETCH_TYPE)) {
return FETCH_TYPE_FULL;
@@ -122,6 +110,27 @@ public class AtlasExportRequest implements Serializable {
return false;
}
+public String getMatchTypeOptionValue() {
+String matchType = null;
+
+if (MapUtils.isNotEmpty(getOptions())) {
+if (getOptions().get(OPTION_ATTR_MATCH_TYPE) != null) {
+matchType =
getOptions().get(OPTION_ATTR_MATCH_TYPE).toString();
+}
+}
+
+return matchType;
+}
+
+public long getChangeTokenFromOptions() {
+if(getFetchTypeOptionValue().equalsIgnoreCase(FETCH_TYPE_INCREMENTAL)
&&
+
getOptions().containsKey(AtlasExportRequest.FETCH_TYPE_INCREMENTAL_CHANGE_MARKER))
{
+return
Long.parseLong(getOptions().get(AtlasExportRequest.FETCH_TYPE_INCREMENTAL_CHANGE_MARKER).toString());
+}
+
+return 0L;
+}
+
public StringBuilder toString(StringBuilder sb) {
if (sb == null) {
sb = new StringBuilder();
http://git-wip-us.apache.org/repos/asf/atlas/blob/dc6be8e2/repository/src/main/java/org/apache/atlas/repository/impexp/ExportService.java
--
diff --git
a/repository/src/main/java/org/apache/atlas/repository/impexp/ExportService.java
b/repository/src/main/java/org/apache/atlas/repository/impexp/ExportService.java
index f10d615..5e972a2 100644
---
a/repository/src/main/java/org/apache/atlas/repository/impexp/ExportService.java
+++
b/repository/src/main/java/org/apache/atlas/repository/impexp/ExportService.java
@@ -70,7 +70,7 @@ import static
org.apache.atlas.model.impexp.AtlasExportRequest.*;
public class ExportService {
private static final Logger LOG =
LoggerFactory.getLogger(ExportService.class);
-private static final String PROPERTY_GUID = "__guid";
+public static final String PROPERTY_GUID = "__guid";
private static final String PROPERTY_IS_PROCESS = "isProcess";
@@ -82,6 +82,8 @@ public class ExportService {
private final AtlasGremlinQueryProvider gremlinQueryProvider;
private ExportTypeProcessor exportTypeProcessor;
private final HdfsPathEntityCreator hdfsPathEntityCreator;
+private IncrementalExportEntityProvider
incrementalExportEntityProvider;
+
@Inject
public ExportService(final AtlasTypeRegistry typeRegistry, AtlasGraph
atlasGraph,
AuditsWriter auditsWriter, HdfsPathEntityCreator
hdfsPathEntityCreator) {
@@ -95,12 +97,12 @@ public class ExportService {
public AtlasExportResult run(ZipSink
[6/8] atlas git commit: ATLAS-2869: Hdfs_path if requested are created and then proceeds with export.
ATLAS-2869: Hdfs_path if requested are created and then proceeds with export.
Signed-off-by: Ashutosh Mestry
Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/c6466004
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/c6466004
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/c6466004
Branch: refs/heads/master
Commit: c6466004619f4e84a2940b0e55022f888fa030b2
Parents: b08b7b4
Author: Ashutosh Mestry
Authored: Tue Sep 11 15:29:02 2018 -0700
Committer: Ashutosh Mestry
Committed: Thu Oct 11 14:26:18 2018 -0700
--
.../atlas/repository/impexp/ExportService.java | 12 +-
.../impexp/HdfsPathEntityCreator.java | 131 +++
.../impexp/HdfsPathEntityCreatorTest.java | 81
3 files changed, 221 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/atlas/blob/c6466004/repository/src/main/java/org/apache/atlas/repository/impexp/ExportService.java
--
diff --git
a/repository/src/main/java/org/apache/atlas/repository/impexp/ExportService.java
b/repository/src/main/java/org/apache/atlas/repository/impexp/ExportService.java
index d3cff78..f10d615 100644
---
a/repository/src/main/java/org/apache/atlas/repository/impexp/ExportService.java
+++
b/repository/src/main/java/org/apache/atlas/repository/impexp/ExportService.java
@@ -81,14 +81,16 @@ public class ExportService {
private final EntityGraphRetriever entityGraphRetriever;
private final AtlasGremlinQueryProvider gremlinQueryProvider;
private ExportTypeProcessor exportTypeProcessor;
-
+private final HdfsPathEntityCreator hdfsPathEntityCreator;
@Inject
-public ExportService(final AtlasTypeRegistry typeRegistry, AtlasGraph
atlasGraph, AuditsWriter auditsWriter) {
+public ExportService(final AtlasTypeRegistry typeRegistry, AtlasGraph
atlasGraph,
+ AuditsWriter auditsWriter, HdfsPathEntityCreator
hdfsPathEntityCreator) {
this.typeRegistry = typeRegistry;
this.entityGraphRetriever = new
EntityGraphRetriever(this.typeRegistry);
this.atlasGraph = atlasGraph;
this.gremlinQueryProvider = AtlasGremlinQueryProvider.INSTANCE;
-this.auditsWriter = auditsWriter;
+this.auditsWriter = auditsWriter;
+this.hdfsPathEntityCreator = hdfsPathEntityCreator;
}
public AtlasExportResult run(ZipSink exportSink, AtlasExportRequest
request, String userName, String hostName,
@@ -244,6 +246,10 @@ public class ExportService {
private List getStartingEntity(AtlasObjectId item, ExportContext
context) throws AtlasBaseException {
List ret = null;
+
if(item.getTypeName().equalsIgnoreCase(HdfsPathEntityCreator.HDFS_PATH_TYPE)) {
+hdfsPathEntityCreator.getCreateEntity(item);
+}
+
if (StringUtils.isNotEmpty(item.getGuid())) {
ret = Collections.singletonList(item.getGuid());
} else if (StringUtils.equalsIgnoreCase(context.matchType,
MATCH_TYPE_FOR_TYPE) && StringUtils.isNotEmpty(item.getTypeName())) {
http://git-wip-us.apache.org/repos/asf/atlas/blob/c6466004/repository/src/main/java/org/apache/atlas/repository/impexp/HdfsPathEntityCreator.java
--
diff --git
a/repository/src/main/java/org/apache/atlas/repository/impexp/HdfsPathEntityCreator.java
b/repository/src/main/java/org/apache/atlas/repository/impexp/HdfsPathEntityCreator.java
new file mode 100644
index 000..fddd60b
--- /dev/null
+++
b/repository/src/main/java/org/apache/atlas/repository/impexp/HdfsPathEntityCreator.java
@@ -0,0 +1,131 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.repository.impexp;
+
+import org.apache.atlas.exception.AtlasBaseException;
+import org.apache.atlas.model.instance.AtlasEntity;
+import org.apache.atlas.model.instance.AtlasObjectId;
+import
[16/17] atlas git commit: ATLAS-2897: Better handling of empty zip files. Unit test fix.
ATLAS-2897: Better handling of empty zip files. Unit test fix.
Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/7ffbec1a
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/7ffbec1a
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/7ffbec1a
Branch: refs/heads/master
Commit: 7ffbec1a2a16f5288fff27d5bb81254d3280a900
Parents: 016eaff
Author: Ashutosh Mestry
Authored: Thu Oct 4 15:30:13 2018 -0700
Committer: Ashutosh Mestry
Committed: Thu Oct 11 17:21:30 2018 -0700
--
.../atlas/repository/impexp/ExportServiceTest.java | 15 ---
1 file changed, 4 insertions(+), 11 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/atlas/blob/7ffbec1a/repository/src/test/java/org/apache/atlas/repository/impexp/ExportServiceTest.java
--
diff --git
a/repository/src/test/java/org/apache/atlas/repository/impexp/ExportServiceTest.java
b/repository/src/test/java/org/apache/atlas/repository/impexp/ExportServiceTest.java
index 7886a64..9f72f1b 100644
---
a/repository/src/test/java/org/apache/atlas/repository/impexp/ExportServiceTest.java
+++
b/repository/src/test/java/org/apache/atlas/repository/impexp/ExportServiceTest.java
@@ -218,7 +218,7 @@ public class ExportServiceTest extends ExportImportTestBase
{
assertNotNull(result.getSourceClusterName());
}
-@Test
+@Test(expectedExceptions = AtlasBaseException.class)
public void requestingEntityNotFound_NoData() throws AtlasBaseException,
IOException {
String requestingIP = "1.0.0.0";
String hostName = "root";
@@ -231,11 +231,7 @@ public class ExportServiceTest extends
ExportImportTestBase {
Assert.assertNull(result.getData());
ByteArrayInputStream bais = new
ByteArrayInputStream(baos.toByteArray());
-ZipSource zipSource = new ZipSource(bais);
-
-assertNotNull(exportService);
-assertNotNull(zipSource.getCreationOrder());
-Assert.assertFalse(zipSource.hasNext());
+new ZipSource(bais);
}
@Test
@@ -306,14 +302,11 @@ public class ExportServiceTest extends
ExportImportTestBase {
AtlasExportResult.OperationStatus.FAIL));
}
-@Test
+@Test(expectedExceptions = AtlasBaseException.class)
public void requestingExportOfNonExistentEntity_ReturnsFailure() throws
Exception {
AtlasExportRequest request = getRequestForEmployee();
tamperEmployeeRequest(request);
-ZipSource zipSource = runExportWithParameters(request);
-
-assertNotNull(zipSource.getCreationOrder());
-assertEquals(zipSource.getCreationOrder().size(), 0);
+runExportWithParameters(request);
}
@Test
[06/17] atlas git commit: ATLAS-2882: refactored import transformer to set context in constructor; fixed incorrect objId match
ATLAS-2882: refactored import transformer to set context in constructor; fixed
incorrect objId match
Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/31c3bea1
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/31c3bea1
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/31c3bea1
Branch: refs/heads/master
Commit: 31c3bea1316d49fe233a061965cef248a97f1168
Parents: 9d4f972
Author: Madhan Neethiraj
Authored: Thu Sep 20 17:27:28 2018 -0700
Committer: Ashutosh Mestry
Committed: Thu Oct 11 17:21:25 2018 -0700
--
.../apache/atlas/entitytransform/Action.java| 150 +
.../entitytransform/AtlasEntityTransformer.java | 27 ++--
.../entitytransform/BaseEntityHandler.java | 138 +---
.../apache/atlas/entitytransform/Condition.java | 141
.../atlas/entitytransform/EntityAttribute.java | 69
.../entitytransform/HdfsPathEntityHandler.java | 28 ++--
.../HiveColumnEntityHandler.java| 28 ++--
.../HiveDatabaseEntityHandler.java | 24 +--
.../HiveStorageDescriptorEntityHandler.java | 26 +--
.../entitytransform/HiveTableEntityHandler.java | 26 +--
.../atlas/entitytransform/NeedsContext.java | 23 ---
.../entitytransform/TransformerContext.java | 8 +-
.../TransformationHandlerTest.java | 160 +++
13 files changed, 470 insertions(+), 378 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/atlas/blob/31c3bea1/intg/src/main/java/org/apache/atlas/entitytransform/Action.java
--
diff --git a/intg/src/main/java/org/apache/atlas/entitytransform/Action.java
b/intg/src/main/java/org/apache/atlas/entitytransform/Action.java
index fa18558..0c6102d 100644
--- a/intg/src/main/java/org/apache/atlas/entitytransform/Action.java
+++ b/intg/src/main/java/org/apache/atlas/entitytransform/Action.java
@@ -22,6 +22,8 @@ import org.apache.atlas.model.instance.AtlasClassification;
import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.model.typedef.AtlasClassificationDef;
import org.apache.atlas.model.typedef.AtlasTypesDef;
+import org.apache.atlas.store.AtlasTypeDefStore;
+import org.apache.atlas.type.AtlasTypeRegistry;
import org.apache.commons.lang.StringUtils;
import
org.apache.atlas.entitytransform.BaseEntityHandler.AtlasTransformableEntity;
import org.slf4j.Logger;
@@ -31,35 +33,35 @@ import java.util.ArrayList;
import java.util.Collections;
+
public abstract class Action {
private static final Logger LOG = LoggerFactory.getLogger(Action.class);
-private static final String ENTITY_KEY = "__entity";
private static final String ACTION_DELIMITER = ":";
-private static final String ACTION_ADD_CLASSIFICATION =
"ADDCLASSIFICATION";
+private static final String ACTION_ADD_CLASSIFICATION =
"ADD_CLASSIFICATION";
private static final String ACTION_NAME_SET= "SET";
private static final String ACTION_NAME_REPLACE_PREFIX = "REPLACE_PREFIX";
private static final String ACTION_NAME_TO_LOWER = "TO_LOWER";
private static final String ACTION_NAME_TO_UPPER = "TO_UPPER";
private static final String ACTION_NAME_CLEAR = "CLEAR";
-protected final String attributeName;
+protected final EntityAttribute attribute;
-protected Action(String attributeName) {
-this.attributeName = attributeName;
+protected Action(EntityAttribute attribute) {
+this.attribute = attribute;
}
-public String getAttributeName() { return attributeName; }
+public EntityAttribute getAttribute() { return attribute; }
public boolean isValid() {
-return StringUtils.isNotEmpty(attributeName);
+return true;
}
public abstract void apply(AtlasTransformableEntity entity);
-public static Action createAction(String key, String value) {
+public static Action createAction(String key, String value,
TransformerContext context) {
if (LOG.isDebugEnabled()) {
LOG.debug("==> Action.createAction(key={}, value={})", key, value);
}
@@ -74,33 +76,35 @@ public abstract class Action {
actionValue = StringUtils.trim(actionValue);
value = StringUtils.trim(value);
+EntityAttribute attribute = new EntityAttribute(StringUtils.trim(key),
context);
+
switch (actionName.toUpperCase()) {
case ACTION_ADD_CLASSIFICATION:
-ret = new AddClassificationAction(actionValue);
-break;
+ret = new AddClassificationAction(attribute, actionValue,
context);
+break;
case ACTION_NAME_REPLACE_PREFIX:
-
[01/17] atlas git commit: ATLAS-2874: Include handling of Atlas Entity Transformers in current Import logic
Repository: atlas
Updated Branches:
refs/heads/master 8746b3063 -> 1eb995434
ATLAS-2874: Include handling of Atlas Entity Transformers in current Import
logic
Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/4b3c078c
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/4b3c078c
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/4b3c078c
Branch: refs/heads/master
Commit: 4b3c078c0efe41d8a5a3fe073bd24c87622a409a
Parents: 8746b30
Author: Sarath Subramanian
Authored: Mon Sep 17 21:57:40 2018 -0700
Committer: Ashutosh Mestry
Committed: Thu Oct 11 15:38:23 2018 -0700
--
.../atlas/model/impexp/AtlasImportRequest.java | 1 +
.../atlas/repository/impexp/ImportService.java | 50 +--
.../atlas/repository/impexp/ZipSource.java | 51
3 files changed, 89 insertions(+), 13 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/atlas/blob/4b3c078c/intg/src/main/java/org/apache/atlas/model/impexp/AtlasImportRequest.java
--
diff --git
a/intg/src/main/java/org/apache/atlas/model/impexp/AtlasImportRequest.java
b/intg/src/main/java/org/apache/atlas/model/impexp/AtlasImportRequest.java
index 2989fbe..06bc231 100644
--- a/intg/src/main/java/org/apache/atlas/model/impexp/AtlasImportRequest.java
+++ b/intg/src/main/java/org/apache/atlas/model/impexp/AtlasImportRequest.java
@@ -40,6 +40,7 @@ public class AtlasImportRequest implements Serializable {
private static final long serialVersionUID = 1L;
public static final String TRANSFORMS_KEY = "transforms";
+public static final String TRANSFORMERS_KEY = "transformers";
public static final String OPTION_KEY_REPLICATED_FROM = "replicatedFrom";
private static final String START_POSITION_KEY = "startPosition";
private static final String START_GUID_KEY = "startGuid";
http://git-wip-us.apache.org/repos/asf/atlas/blob/4b3c078c/repository/src/main/java/org/apache/atlas/repository/impexp/ImportService.java
--
diff --git
a/repository/src/main/java/org/apache/atlas/repository/impexp/ImportService.java
b/repository/src/main/java/org/apache/atlas/repository/impexp/ImportService.java
index a88ba2b..a09385e 100644
---
a/repository/src/main/java/org/apache/atlas/repository/impexp/ImportService.java
+++
b/repository/src/main/java/org/apache/atlas/repository/impexp/ImportService.java
@@ -19,15 +19,17 @@ package org.apache.atlas.repository.impexp;
import com.google.common.annotations.VisibleForTesting;
import org.apache.atlas.AtlasErrorCode;
+import org.apache.atlas.entitytransform.BaseEntityHandler;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.impexp.AtlasImportRequest;
import org.apache.atlas.model.impexp.AtlasImportResult;
+import org.apache.atlas.model.impexp.AttributeTransform;
import org.apache.atlas.model.typedef.AtlasTypesDef;
import org.apache.atlas.repository.store.graph.BulkImporter;
import org.apache.atlas.store.AtlasTypeDefStore;
-import org.apache.atlas.type.AtlasEntityType;
import org.apache.atlas.type.AtlasType;
import org.apache.atlas.type.AtlasTypeRegistry;
+import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.MapUtils;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.StringUtils;
@@ -40,6 +42,11 @@ import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import static
org.apache.atlas.model.impexp.AtlasImportRequest.TRANSFORMERS_KEY;
+import static org.apache.atlas.model.impexp.AtlasImportRequest.TRANSFORMS_KEY;
@Component
public class ImportService {
@@ -82,9 +89,12 @@ public class ImportService {
try {
LOG.info("==> import(user={}, from={}, request={})", userName,
requestingIP, request);
-String transforms = MapUtils.isNotEmpty(request.getOptions()) ?
request.getOptions().get(AtlasImportRequest.TRANSFORMS_KEY) : null;
-
+String transforms = MapUtils.isNotEmpty(request.getOptions()) ?
request.getOptions().get(TRANSFORMS_KEY) : null;
setImportTransform(source, transforms);
+
+String transformers = MapUtils.isNotEmpty(request.getOptions()) ?
request.getOptions().get(TRANSFORMERS_KEY) : null;
+setEntityTransformerHandlers(source, transformers);
+
startTimestamp = System.currentTimeMillis();
processTypes(source.getTypesDef(), result);
setStartPosition(request, source);
@@ -121,6 +131,38 @@ public class ImportService {
}
+
[13/17] atlas git commit: ATLAS-2897: Better handling of empty zip files.
ATLAS-2897: Better handling of empty zip files.
Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/016eaffa
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/016eaffa
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/016eaffa
Branch: refs/heads/master
Commit: 016eaffae5518b86bba7bfdda4fc720749362a93
Parents: 8a26c79
Author: Ashutosh Mestry
Authored: Thu Oct 4 14:38:21 2018 -0700
Committer: Ashutosh Mestry
Committed: Thu Oct 11 17:21:29 2018 -0700
--
.../atlas/repository/impexp/AtlasServerService.java | 1 -
.../org/apache/atlas/repository/impexp/ZipSource.java | 12
2 files changed, 8 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/atlas/blob/016eaffa/repository/src/main/java/org/apache/atlas/repository/impexp/AtlasServerService.java
--
diff --git
a/repository/src/main/java/org/apache/atlas/repository/impexp/AtlasServerService.java
b/repository/src/main/java/org/apache/atlas/repository/impexp/AtlasServerService.java
index a3489a8..13a8cd9 100644
---
a/repository/src/main/java/org/apache/atlas/repository/impexp/AtlasServerService.java
+++
b/repository/src/main/java/org/apache/atlas/repository/impexp/AtlasServerService.java
@@ -65,7 +65,6 @@ public class AtlasServerService {
try {
return dataAccess.load(server);
} catch (AtlasBaseException e) {
-LOG.error("dataAccess", e);
throw e;
}
}
http://git-wip-us.apache.org/repos/asf/atlas/blob/016eaffa/repository/src/main/java/org/apache/atlas/repository/impexp/ZipSource.java
--
diff --git
a/repository/src/main/java/org/apache/atlas/repository/impexp/ZipSource.java
b/repository/src/main/java/org/apache/atlas/repository/impexp/ZipSource.java
index be8c168..bf23f81 100644
--- a/repository/src/main/java/org/apache/atlas/repository/impexp/ZipSource.java
+++ b/repository/src/main/java/org/apache/atlas/repository/impexp/ZipSource.java
@@ -72,10 +72,14 @@ public class ZipSource implements EntityImportStream {
}
private boolean isZipFileEmpty() {
-return MapUtils.isEmpty(guidEntityJsonMap) ||
-
(guidEntityJsonMap.containsKey(ZipExportFileNames.ATLAS_EXPORT_ORDER_NAME.toString())
&&
-
(guidEntityJsonMap.get(ZipExportFileNames.ATLAS_EXPORT_ORDER_NAME.toString())
== null)
-);
+if (MapUtils.isEmpty(guidEntityJsonMap)) {
+return true;
+}
+
+String key = ZipExportFileNames.ATLAS_EXPORT_ORDER_NAME.toString();
+return (guidEntityJsonMap.containsKey(key) &&
+ StringUtils.isNotEmpty(guidEntityJsonMap.get(key)) &&
+ guidEntityJsonMap.get(key).equals("[]"));
}
public ImportTransforms getImportTransform() { return
this.importTransform; }
[15/17] atlas git commit: ATLAS-2906: Allow transforms to be applied when entity-level transforms are present.
ATLAS-2906: Allow transforms to be applied when entity-level transforms are
present.
Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/8a26c79e
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/8a26c79e
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/8a26c79e
Branch: refs/heads/master
Commit: 8a26c79e2f9ecfc14052c0954cb6e66308af09c6
Parents: 7763fd0
Author: Ashutosh Mestry
Authored: Wed Oct 3 18:08:51 2018 -0700
Committer: Ashutosh Mestry
Committed: Thu Oct 11 17:21:29 2018 -0700
--
.../entitytransform/HiveStorageDescriptorEntityHandler.java | 9 -
.../atlas/entitytransform/TransformationConstants.java | 1 +
2 files changed, 9 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/atlas/blob/8a26c79e/intg/src/main/java/org/apache/atlas/entitytransform/HiveStorageDescriptorEntityHandler.java
--
diff --git
a/intg/src/main/java/org/apache/atlas/entitytransform/HiveStorageDescriptorEntityHandler.java
b/intg/src/main/java/org/apache/atlas/entitytransform/HiveStorageDescriptorEntityHandler.java
index dc4edfb..38de206 100644
---
a/intg/src/main/java/org/apache/atlas/entitytransform/HiveStorageDescriptorEntityHandler.java
+++
b/intg/src/main/java/org/apache/atlas/entitytransform/HiveStorageDescriptorEntityHandler.java
@@ -26,7 +26,7 @@ import java.util.List;
import static org.apache.atlas.entitytransform.TransformationConstants.*;
public class HiveStorageDescriptorEntityHandler extends BaseEntityHandler {
-static final List CUSTOM_TRANSFORM_ATTRIBUTES =
Arrays.asList(HIVE_DB_NAME_ATTRIBUTE, HIVE_TABLE_NAME_ATTRIBUTE,
HIVE_DB_CLUSTER_NAME_ATTRIBUTE);
+static final List CUSTOM_TRANSFORM_ATTRIBUTES =
Arrays.asList(HIVE_DB_NAME_ATTRIBUTE, HIVE_TABLE_NAME_ATTRIBUTE,
HIVE_DB_CLUSTER_NAME_ATTRIBUTE, HIVE_STORAGE_DESC_LOCATION_ATTRIBUTE);
public HiveStorageDescriptorEntityHandler(List
transformers) {
@@ -90,6 +90,9 @@ public class HiveStorageDescriptorEntityHandler extends
BaseEntityHandler {
case HIVE_DB_CLUSTER_NAME_ATTRIBUTE:
return clusterName;
+
+case HIVE_STORAGE_DESC_LOCATION_ATTRIBUTE:
+return location;
}
return super.getAttribute(attribute);
@@ -116,6 +119,10 @@ public class HiveStorageDescriptorEntityHandler extends
BaseEntityHandler {
isCustomAttributeUpdated = true;
break;
+case HIVE_STORAGE_DESC_LOCATION_ATTRIBUTE:
+location = attributeValue;
+break;
+
default:
super.setAttribute(attribute, attributeValue);
break;
http://git-wip-us.apache.org/repos/asf/atlas/blob/8a26c79e/intg/src/main/java/org/apache/atlas/entitytransform/TransformationConstants.java
--
diff --git
a/intg/src/main/java/org/apache/atlas/entitytransform/TransformationConstants.java
b/intg/src/main/java/org/apache/atlas/entitytransform/TransformationConstants.java
index 51c3ace..247de73 100644
---
a/intg/src/main/java/org/apache/atlas/entitytransform/TransformationConstants.java
+++
b/intg/src/main/java/org/apache/atlas/entitytransform/TransformationConstants.java
@@ -37,6 +37,7 @@ public final class TransformationConstants {
public static final String HDFS_PATH_NAME_ATTRIBUTE =
"hdfs_path.name";
public static final String HDFS_PATH_PATH_ATTRIBUTE =
"hdfs_path.path";
public static final String HDFS_CLUSTER_NAME_ATTRIBUTE=
"hdfs_path.clusterName";
+public static final String HIVE_STORAGE_DESC_LOCATION_ATTRIBUTE =
"hive_storagedesc.location";
public static final char TYPE_NAME_ATTRIBUTE_NAME_SEP = '.';
public static final char CLUSTER_DELIMITER= '@';
[05/17] atlas git commit: ATLAS-2886: Support for fully qualified server name
ATLAS-2886: Support for fully qualified server name
Signed-off-by: Madhan Neethiraj
Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/8639ada6
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/8639ada6
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/8639ada6
Branch: refs/heads/master
Commit: 8639ada6a74cdaa32b0b493aaff1168733487eef
Parents: 31c3bea
Author: Ashutosh Mestry
Authored: Sun Sep 23 08:50:06 2018 -0700
Committer: Ashutosh Mestry
Committed: Thu Oct 11 17:21:25 2018 -0700
--
addons/models/0010-base_model.json | 335 +++
.../apache/atlas/entitytransform/Condition.java | 2 -
.../atlas/repository/impexp/AuditsWriter.java | 44 ++-
.../impexp/ExportImportAuditServiceTest.java| 2 +-
.../IncrementalExportEntityProviderTest.java| 2 -
.../impexp/ReplicationEntityAttributeTest.java | 14 +-
.../stocksDB-Entities/export-replicatedTo.json | 2 +-
.../import-replicatedFrom.json | 2 +-
8 files changed, 377 insertions(+), 26 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/atlas/blob/8639ada6/addons/models/0010-base_model.json
--
diff --git a/addons/models/0010-base_model.json
b/addons/models/0010-base_model.json
new file mode 100644
index 000..1bfbf2f
--- /dev/null
+++ b/addons/models/0010-base_model.json
@@ -0,0 +1,335 @@
+{
+ "enumDefs": [],
+ "structDefs": [],
+ "classificationDefs": [
+{
+ "name": "TaxonomyTerm",
+ "superTypes": [],
+ "typeVersion": "1.0",
+ "attributeDefs": [
+{
+ "name": "atlas.taxonomy",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+}
+ ]
+}
+ ],
+ "entityDefs": [
+{
+ "name": "Referenceable",
+ "superTypes": [],
+ "typeVersion": "1.0",
+ "attributeDefs": [
+{
+ "name": "qualifiedName",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": true,
+ "isOptional": false,
+ "isUnique": true
+}
+ ]
+},
+{
+ "name": "__internal",
+ "superTypes": [],
+ "typeVersion": "1.0",
+ "attributeDefs": []
+},
+{
+ "name": "Asset",
+ "superTypes": [],
+ "typeVersion": "1.0",
+ "attributeDefs": [
+{
+ "name": "name",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": true,
+ "isOptional": false,
+ "isUnique": false
+},
+{
+ "name": "description",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+},
+{
+ "name": "owner",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": true,
+ "isOptional": true,
+ "isUnique": false
+}
+ ]
+},
+{
+ "name": "DataSet",
+ "superTypes": [
+"Referenceable",
+"Asset"
+ ],
+ "typeVersion": "1.0",
+ "attributeDefs": []
+},
+{
+ "name": "Infrastructure",
+ "superTypes": [
+"Referenceable",
+"Asset"
+ ],
+ "typeVersion": "1.0",
+ "attributeDefs": []
+},
+{
+ "name": "Process",
+ "superTypes": [
+"Referenceable",
+"Asset"
+ ],
+ "typeVersion": "1.0",
+ "attributeDefs": [
+{
+ "name": "inputs",
+ "typeName": "array",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+},
+{
+ "name": "outputs",
+ "typeName": "array",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+}
+ ]
+},
+{
+ "name": "AtlasServer",
+ "typeVersion": "1.0",
+ "superTypes": [
+ ],
+ "attributeDefs": [
+{
+ "name": "name",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": true,
+ "isOptional": false,
+ "isUnique": false
+},
+{
+ "name": "displayName",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": true,
+ "isOptional": false,
+ "isUnique": false
+},
+{
+ "name": "fullName",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": true,
+
[12/17] atlas git commit: ATLAS-2888: Change marker fix for server name. Unit test fix.
ATLAS-2888: Change marker fix for server name. Unit test fix.
Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/f6acb086
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/f6acb086
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/f6acb086
Branch: refs/heads/master
Commit: f6acb086146976ea2a63d93648be8a39d7acbc53
Parents: 6f74720
Author: Ashutosh Mestry
Authored: Mon Oct 1 22:08:30 2018 -0700
Committer: Ashutosh Mestry
Committed: Thu Oct 11 17:21:28 2018 -0700
--
.../atlas/repository/impexp/ExportServiceTest.java | 16 ++--
.../atlas/repository/impexp/ImportServiceTest.java | 10 +-
.../impexp/ReplicationEntityAttributeTest.java | 4 ++--
.../ClassificationPropagationTest.java | 2 +-
4 files changed, 18 insertions(+), 14 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/atlas/blob/f6acb086/repository/src/test/java/org/apache/atlas/repository/impexp/ExportServiceTest.java
--
diff --git
a/repository/src/test/java/org/apache/atlas/repository/impexp/ExportServiceTest.java
b/repository/src/test/java/org/apache/atlas/repository/impexp/ExportServiceTest.java
index 377bd67..7aa0b57 100644
---
a/repository/src/test/java/org/apache/atlas/repository/impexp/ExportServiceTest.java
+++
b/repository/src/test/java/org/apache/atlas/repository/impexp/ExportServiceTest.java
@@ -26,6 +26,7 @@ import org.apache.atlas.model.impexp.AtlasExportRequest;
import org.apache.atlas.model.impexp.AtlasExportResult;
import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.model.instance.AtlasObjectId;
+import org.apache.atlas.model.instance.EntityMutationResponse;
import org.apache.atlas.model.typedef.AtlasTypesDef;
import org.apache.atlas.repository.graph.AtlasGraphProvider;
import
org.apache.atlas.repository.store.bootstrap.AtlasTypeDefStoreInitializer;
@@ -84,9 +85,11 @@ public class ExportServiceTest extends ExportImportTestBase {
@Inject
private ExportImportAuditService auditService;
+@Inject
+private AtlasEntityStoreV2 entityStore;
+
private DeleteHandlerV1 deleteHandler = mock(SoftDeleteHandlerV1.class);;
private AtlasEntityChangeNotifier mockChangeNotifier =
mock(AtlasEntityChangeNotifier.class);
-private AtlasEntityStoreV2entityStore;
@BeforeTest
public void setupTest() throws IOException, AtlasBaseException {
@@ -106,11 +109,12 @@ public class ExportServiceTest extends
ExportImportTestBase {
typeDefStore.createTypesDef(typesToCreate);
}
-AtlasEntity.AtlasEntitiesWithExtInfo hrDept =
TestUtilsV2.createDeptEg2();
-
-AtlasEntityStream entityStream = new AtlasEntityStream(hrDept);
-entityStore.createOrUpdate(entityStream, false);
-LOG.debug("==> setupSampleData: ",
AtlasEntity.dumpObjects(hrDept.getEntities(), null).toString());
+AtlasEntity.AtlasEntitiesWithExtInfo deptEg2 =
TestUtilsV2.createDeptEg2();
+AtlasEntityStream entityStream = new AtlasEntityStream(deptEg2);
+EntityMutationResponse emr = entityStore.createOrUpdate(entityStream,
false);
+assertNotNull(emr);
+assertNotNull(emr.getCreatedEntities());
+assertTrue(emr.getCreatedEntities().size() > 0);
}
@AfterClass
http://git-wip-us.apache.org/repos/asf/atlas/blob/f6acb086/repository/src/test/java/org/apache/atlas/repository/impexp/ImportServiceTest.java
--
diff --git
a/repository/src/test/java/org/apache/atlas/repository/impexp/ImportServiceTest.java
b/repository/src/test/java/org/apache/atlas/repository/impexp/ImportServiceTest.java
index e0bbb11..a1d6cef 100644
---
a/repository/src/test/java/org/apache/atlas/repository/impexp/ImportServiceTest.java
+++
b/repository/src/test/java/org/apache/atlas/repository/impexp/ImportServiceTest.java
@@ -213,7 +213,7 @@ public class ImportServiceTest extends ExportImportTestBase
{
}
@DataProvider(name = "stocks-legacy")
-public static Object[][] getDataFromLegacyStocks(ITestContext context)
throws IOException {
+public static Object[][] getDataFromLegacyStocks(ITestContext context)
throws IOException, AtlasBaseException {
return getZipSource("stocks.zip");
}
@@ -254,7 +254,7 @@ public class ImportServiceTest extends ExportImportTestBase
{
}
@DataProvider(name = "stocks-glossary")
-public static Object[][] getDataFromGlossary(ITestContext context) throws
IOException {
+public static Object[][] getDataFromGlossary(ITestContext context) throws
IOException, AtlasBaseException {
return getZipSource("stocks-glossary.zip");
}
@@ -298,12 +298,12 @@ public
[14/17] atlas git commit: ATLAS-2897: Elegant handling of empty zip files. Part 2.
ATLAS-2897: Elegant handling of empty zip files. Part 2.
Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/7763fd0d
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/7763fd0d
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/7763fd0d
Branch: refs/heads/master
Commit: 7763fd0d329e5995ff9dc4c1f7bf73099eb85c27
Parents: f6acb08
Author: Ashutosh Mestry
Authored: Wed Oct 3 09:01:07 2018 -0700
Committer: Ashutosh Mestry
Committed: Thu Oct 11 17:21:29 2018 -0700
--
.../atlas/repository/impexp/ZipSource.java | 19 +++
.../repository/impexp/ExportServiceTest.java| 1 -
.../impexp/ZipFileResourceTestUtils.java| 20
.../atlas/web/resources/AdminResource.java | 19 ++-
4 files changed, 37 insertions(+), 22 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/atlas/blob/7763fd0d/repository/src/main/java/org/apache/atlas/repository/impexp/ZipSource.java
--
diff --git
a/repository/src/main/java/org/apache/atlas/repository/impexp/ZipSource.java
b/repository/src/main/java/org/apache/atlas/repository/impexp/ZipSource.java
index bfa0441..be8c168 100644
--- a/repository/src/main/java/org/apache/atlas/repository/impexp/ZipSource.java
+++ b/repository/src/main/java/org/apache/atlas/repository/impexp/ZipSource.java
@@ -64,13 +64,20 @@ public class ZipSource implements EntityImportStream {
this.importTransform = importTransform;
updateGuidZipEntryMap();
-if (MapUtils.isEmpty(guidEntityJsonMap)) {
+if (isZipFileEmpty()) {
throw new AtlasBaseException(IMPORT_ATTEMPTING_EMPTY_ZIP,
"Attempting to import empty ZIP.");
}
setCreationOrder();
}
+private boolean isZipFileEmpty() {
+return MapUtils.isEmpty(guidEntityJsonMap) ||
+
(guidEntityJsonMap.containsKey(ZipExportFileNames.ATLAS_EXPORT_ORDER_NAME.toString())
&&
+
(guidEntityJsonMap.get(ZipExportFileNames.ATLAS_EXPORT_ORDER_NAME.toString())
== null)
+);
+}
+
public ImportTransforms getImportTransform() { return
this.importTransform; }
public void setImportTransform(ImportTransforms importTransform) {
@@ -136,7 +143,7 @@ public class ZipSource implements EntityImportStream {
zipInputStream.close();
}
-public List getCreationOrder() throws AtlasBaseException {
+public List getCreationOrder() {
return this.creationOrder;
}
@@ -234,12 +241,8 @@ public class ZipSource implements EntityImportStream {
@Override
public void reset() {
-try {
-getCreationOrder();
-this.iterator = this.creationOrder.iterator();
-} catch (AtlasBaseException e) {
-LOG.error("reset", e);
-}
+getCreationOrder();
+this.iterator = this.creationOrder.iterator();
}
@Override
http://git-wip-us.apache.org/repos/asf/atlas/blob/7763fd0d/repository/src/test/java/org/apache/atlas/repository/impexp/ExportServiceTest.java
--
diff --git
a/repository/src/test/java/org/apache/atlas/repository/impexp/ExportServiceTest.java
b/repository/src/test/java/org/apache/atlas/repository/impexp/ExportServiceTest.java
index 7aa0b57..7886a64 100644
---
a/repository/src/test/java/org/apache/atlas/repository/impexp/ExportServiceTest.java
+++
b/repository/src/test/java/org/apache/atlas/repository/impexp/ExportServiceTest.java
@@ -314,7 +314,6 @@ public class ExportServiceTest extends ExportImportTestBase
{
assertNotNull(zipSource.getCreationOrder());
assertEquals(zipSource.getCreationOrder().size(), 0);
-assertEquals(AtlasExportResult.OperationStatus.FAIL,
zipSource.getExportResult().getOperationStatus());
}
@Test
http://git-wip-us.apache.org/repos/asf/atlas/blob/7763fd0d/repository/src/test/java/org/apache/atlas/repository/impexp/ZipFileResourceTestUtils.java
--
diff --git
a/repository/src/test/java/org/apache/atlas/repository/impexp/ZipFileResourceTestUtils.java
b/repository/src/test/java/org/apache/atlas/repository/impexp/ZipFileResourceTestUtils.java
index fe473b8..5e287d8 100644
---
a/repository/src/test/java/org/apache/atlas/repository/impexp/ZipFileResourceTestUtils.java
+++
b/repository/src/test/java/org/apache/atlas/repository/impexp/ZipFileResourceTestUtils.java
@@ -255,19 +255,15 @@ public class ZipFileResourceTestUtils {
public static AtlasEntity.AtlasEntityWithExtInfo getEntities(ZipSource
source, int expectedCount) {
AtlasEntity.AtlasEntityWithExtInfo
[09/17] atlas git commit: ATLAS-2895: Server full name processing
ATLAS-2895: Server full name processing
Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/84c6fb2b
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/84c6fb2b
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/84c6fb2b
Branch: refs/heads/master
Commit: 84c6fb2bc3c6ec68315ed522027a566bfed8f678
Parents: f4dac18
Author: Ashutosh Mestry
Authored: Wed Sep 26 14:24:13 2018 -0700
Committer: Ashutosh Mestry
Committed: Thu Oct 11 17:21:27 2018 -0700
--
.../apache/atlas/model/impexp/AtlasExportRequest.java | 14 --
.../apache/atlas/repository/impexp/AuditsWriter.java | 9 -
.../impexp/ReplicationEntityAttributeTest.java| 14 +-
3 files changed, 29 insertions(+), 8 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/atlas/blob/84c6fb2b/intg/src/main/java/org/apache/atlas/model/impexp/AtlasExportRequest.java
--
diff --git
a/intg/src/main/java/org/apache/atlas/model/impexp/AtlasExportRequest.java
b/intg/src/main/java/org/apache/atlas/model/impexp/AtlasExportRequest.java
index e78bb53..8fb7c68 100644
--- a/intg/src/main/java/org/apache/atlas/model/impexp/AtlasExportRequest.java
+++ b/intg/src/main/java/org/apache/atlas/model/impexp/AtlasExportRequest.java
@@ -81,7 +81,7 @@ public class AtlasExportRequest implements Serializable {
}
public String getFetchTypeOptionValue() {
-if(getOptions() == null ||
!getOptions().containsKey(OPTION_FETCH_TYPE)) {
+if(MapUtils.isEmpty(getOptions()) ||
!getOptions().containsKey(OPTION_FETCH_TYPE)) {
return FETCH_TYPE_FULL;
}
@@ -94,7 +94,8 @@ public class AtlasExportRequest implements Serializable {
}
public boolean getSkipLineageOptionValue() {
-if(!getOptions().containsKey(AtlasExportRequest.OPTION_SKIP_LINEAGE)) {
+if(MapUtils.isEmpty(getOptions()) ||
+
!getOptions().containsKey(AtlasExportRequest.OPTION_SKIP_LINEAGE)) {
return false;
}
@@ -123,12 +124,13 @@ public class AtlasExportRequest implements Serializable {
}
public long getChangeTokenFromOptions() {
-if(getFetchTypeOptionValue().equalsIgnoreCase(FETCH_TYPE_INCREMENTAL)
&&
-
getOptions().containsKey(AtlasExportRequest.FETCH_TYPE_INCREMENTAL_CHANGE_MARKER))
{
-return
Long.parseLong(getOptions().get(AtlasExportRequest.FETCH_TYPE_INCREMENTAL_CHANGE_MARKER).toString());
+if (MapUtils.isEmpty(getOptions()) ||
+
!getFetchTypeOptionValue().equalsIgnoreCase(FETCH_TYPE_INCREMENTAL) ||
+
!getOptions().containsKey(AtlasExportRequest.FETCH_TYPE_INCREMENTAL_CHANGE_MARKER))
{
+return 0L;
}
-return 0L;
+return
Long.parseLong(getOptions().get(AtlasExportRequest.FETCH_TYPE_INCREMENTAL_CHANGE_MARKER).toString());
}
public StringBuilder toString(StringBuilder sb) {
http://git-wip-us.apache.org/repos/asf/atlas/blob/84c6fb2b/repository/src/main/java/org/apache/atlas/repository/impexp/AuditsWriter.java
--
diff --git
a/repository/src/main/java/org/apache/atlas/repository/impexp/AuditsWriter.java
b/repository/src/main/java/org/apache/atlas/repository/impexp/AuditsWriter.java
index cc10660..7737197 100644
---
a/repository/src/main/java/org/apache/atlas/repository/impexp/AuditsWriter.java
+++
b/repository/src/main/java/org/apache/atlas/repository/impexp/AuditsWriter.java
@@ -127,7 +127,14 @@ public class AuditsWriter {
return fullName;
}
-return StringUtils.split(fullName, "$")[1];
+String[] splits = StringUtils.split(fullName,
DC_SERVER_NAME_SEPARATOR);
+if (splits == null || splits.length < 1) {
+return "";
+} else if (splits.length >= 2) {
+return splits[1];
+} else {
+return splits[0];
+}
}
private void saveCurrentServer() throws AtlasBaseException {
http://git-wip-us.apache.org/repos/asf/atlas/blob/84c6fb2b/repository/src/test/java/org/apache/atlas/repository/impexp/ReplicationEntityAttributeTest.java
--
diff --git
a/repository/src/test/java/org/apache/atlas/repository/impexp/ReplicationEntityAttributeTest.java
b/repository/src/test/java/org/apache/atlas/repository/impexp/ReplicationEntityAttributeTest.java
index 94483f5..81b9106 100644
---
a/repository/src/test/java/org/apache/atlas/repository/impexp/ReplicationEntityAttributeTest.java
+++
b/repository/src/test/java/org/apache/atlas/repository/impexp/ReplicationEntityAttributeTest.java
@@ -123,7 +123,19 @@ public class
[17/17] atlas git commit: ATLAS-2909: ChangeMarker updated during initialization.
ATLAS-2909: ChangeMarker updated during initialization.
Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/1eb99543
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/1eb99543
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/1eb99543
Branch: refs/heads/master
Commit: 1eb995434cb00d228eb403f47799fadc8dd58c08
Parents: 7ffbec1
Author: Ashutosh Mestry
Authored: Sun Oct 7 22:59:00 2018 -0700
Committer: Ashutosh Mestry
Committed: Thu Oct 11 17:21:31 2018 -0700
--
.../apache/atlas/repository/impexp/AtlasServerService.java | 9 ++---
.../repository/store/graph/v2/AtlasTypeDefGraphStoreV2.java | 2 +-
.../apache/atlas/repository/impexp/ExportServiceTest.java | 4 +---
.../src/main/java/org/apache/atlas/RequestContext.java | 5 +
4 files changed, 9 insertions(+), 11 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/atlas/blob/1eb99543/repository/src/main/java/org/apache/atlas/repository/impexp/AtlasServerService.java
--
diff --git
a/repository/src/main/java/org/apache/atlas/repository/impexp/AtlasServerService.java
b/repository/src/main/java/org/apache/atlas/repository/impexp/AtlasServerService.java
index 13a8cd9..8859a9d 100644
---
a/repository/src/main/java/org/apache/atlas/repository/impexp/AtlasServerService.java
+++
b/repository/src/main/java/org/apache/atlas/repository/impexp/AtlasServerService.java
@@ -88,13 +88,8 @@ public class AtlasServerService {
}
@GraphTransaction
-public AtlasServer save(AtlasServer server) {
-
-try {
-return dataAccess.save(server);
-} catch (AtlasBaseException e) {
-return server;
-}
+public AtlasServer save(AtlasServer server) throws AtlasBaseException {
+ return dataAccess.save(server);
}
@GraphTransaction
http://git-wip-us.apache.org/repos/asf/atlas/blob/1eb99543/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasTypeDefGraphStoreV2.java
--
diff --git
a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasTypeDefGraphStoreV2.java
b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasTypeDefGraphStoreV2.java
index bcdc7a8..3421331 100644
---
a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasTypeDefGraphStoreV2.java
+++
b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasTypeDefGraphStoreV2.java
@@ -524,7 +524,7 @@ public class AtlasTypeDefGraphStoreV2 extends
AtlasTypeDefGraphStore {
}
private String getCurrentUser() {
-String ret = RequestContext.get().getUser();
+String ret = RequestContext.getCurrentUser();
if (StringUtils.isBlank(ret)) {
ret = System.getProperty("user.name");
http://git-wip-us.apache.org/repos/asf/atlas/blob/1eb99543/repository/src/test/java/org/apache/atlas/repository/impexp/ExportServiceTest.java
--
diff --git
a/repository/src/test/java/org/apache/atlas/repository/impexp/ExportServiceTest.java
b/repository/src/test/java/org/apache/atlas/repository/impexp/ExportServiceTest.java
index 9f72f1b..18e7c41 100644
---
a/repository/src/test/java/org/apache/atlas/repository/impexp/ExportServiceTest.java
+++
b/repository/src/test/java/org/apache/atlas/repository/impexp/ExportServiceTest.java
@@ -95,13 +95,11 @@ public class ExportServiceTest extends ExportImportTestBase
{
public void setupTest() throws IOException, AtlasBaseException {
RequestContext.clear();
RequestContext.get().setUser(TestUtilsV2.TEST_USER, null);
-ZipFileResourceTestUtils.loadBaseModel(typeDefStore, typeRegistry);
+basicSetup(typeDefStore, typeRegistry);
}
@BeforeClass
public void setupSampleData() throws AtlasBaseException {
-entityStore = new AtlasEntityStoreV2(deleteHandler, typeRegistry,
mockChangeNotifier, graphMapper);;
-
AtlasTypesDef sampleTypes = TestUtilsV2.defineDeptEmployeeTypes();
AtlasTypesDef typesToCreate =
AtlasTypeDefStoreInitializer.getTypesToCreate(sampleTypes, typeRegistry);
http://git-wip-us.apache.org/repos/asf/atlas/blob/1eb99543/server-api/src/main/java/org/apache/atlas/RequestContext.java
--
diff --git a/server-api/src/main/java/org/apache/atlas/RequestContext.java
b/server-api/src/main/java/org/apache/atlas/RequestContext.java
index 25a35ce..9a9bba6 100644
--- a/server-api/src/main/java/org/apache/atlas/RequestContext.java
+++ b/server-api/src/main/java/org/apache/atlas/RequestContext.java
@@ -93,6 +93,11 @@ public
[11/17] atlas git commit: ATLAS-2888: Change marker fix for server name
ATLAS-2888: Change marker fix for server name
Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/6f747203
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/6f747203
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/6f747203
Branch: refs/heads/master
Commit: 6f74720332d1dfb98c2b3da79516a17a37b5db1c
Parents: 6787141
Author: Ashutosh Mestry
Authored: Mon Oct 1 16:07:06 2018 -0700
Committer: Ashutosh Mestry
Committed: Thu Oct 11 17:21:28 2018 -0700
--
.../atlas/repository/impexp/AtlasServerService.java | 4 ++--
.../apache/atlas/repository/impexp/AuditsWriter.java | 14 --
2 files changed, 6 insertions(+), 12 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/atlas/blob/6f747203/repository/src/main/java/org/apache/atlas/repository/impexp/AtlasServerService.java
--
diff --git
a/repository/src/main/java/org/apache/atlas/repository/impexp/AtlasServerService.java
b/repository/src/main/java/org/apache/atlas/repository/impexp/AtlasServerService.java
index 0761441..a3489a8 100644
---
a/repository/src/main/java/org/apache/atlas/repository/impexp/AtlasServerService.java
+++
b/repository/src/main/java/org/apache/atlas/repository/impexp/AtlasServerService.java
@@ -70,8 +70,8 @@ public class AtlasServerService {
}
}
-public AtlasServer getCreateAtlasServer(String name, String fullName)
throws AtlasBaseException {
-AtlasServer defaultServer = new AtlasServer(name, fullName);
+public AtlasServer getCreateAtlasServer(String clusterName, String
serverFullName) throws AtlasBaseException {
+AtlasServer defaultServer = new AtlasServer(clusterName,
serverFullName);
AtlasServer server = getAtlasServer(defaultServer);
if (server == null) {
return save(defaultServer);
http://git-wip-us.apache.org/repos/asf/atlas/blob/6f747203/repository/src/main/java/org/apache/atlas/repository/impexp/AuditsWriter.java
--
diff --git
a/repository/src/main/java/org/apache/atlas/repository/impexp/AuditsWriter.java
b/repository/src/main/java/org/apache/atlas/repository/impexp/AuditsWriter.java
index 7737197..3612c45 100644
---
a/repository/src/main/java/org/apache/atlas/repository/impexp/AuditsWriter.java
+++
b/repository/src/main/java/org/apache/atlas/repository/impexp/AuditsWriter.java
@@ -93,18 +93,12 @@ public class AuditsWriter {
: StringUtils.EMPTY;
}
-private AtlasServer saveServer(String name, String serverFullName) {
-AtlasServer cluster = new AtlasServer(name, serverFullName);
-return atlasServerService.save(cluster);
-}
-
-private AtlasServer saveServer(String name, String serverFullName,
+private AtlasServer saveServer(String clusterName, String serverFullName,
String entityGuid,
- long lastModifiedTimestamp) {
+ long lastModifiedTimestamp) throws
AtlasBaseException {
-AtlasServer server = new AtlasServer(name, serverFullName);
+AtlasServer server =
atlasServerService.getCreateAtlasServer(clusterName, serverFullName);
server.setAdditionalInfoRepl(entityGuid, lastModifiedTimestamp);
-
if (LOG.isDebugEnabled()) {
LOG.debug("saveServer: {}", server);
}
@@ -138,7 +132,7 @@ public class AuditsWriter {
}
private void saveCurrentServer() throws AtlasBaseException {
-saveServer(getCurrentClusterName(), getCurrentClusterName());
+atlasServerService.getCreateAtlasServer(getCurrentClusterName(),
getCurrentClusterName());
}
private class ExportAudits {
[08/17] atlas git commit: ATLAS-2897: Elegant handling of empty zip files.
ATLAS-2897: Elegant handling of empty zip files.
Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/5fe6d830
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/5fe6d830
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/5fe6d830
Branch: refs/heads/master
Commit: 5fe6d8306da60bc20dbd8c6e51a9040ea32e9ebb
Parents: 84c6fb2
Author: Ashutosh Mestry
Authored: Thu Sep 27 09:27:30 2018 -0700
Committer: Ashutosh Mestry
Committed: Thu Oct 11 17:21:27 2018 -0700
--
.../java/org/apache/atlas/AtlasErrorCode.java | 1 +
.../atlas/repository/impexp/ZipSource.java | 21 +++-
.../repository/impexp/ImportServiceTest.java| 24 +--
.../impexp/ZipFileResourceTestUtils.java| 6 ++---
.../atlas/repository/impexp/ZipSourceTest.java | 8 +++
repository/src/test/resources/empty.zip | Bin 0 -> 22 bytes
6 files changed, 40 insertions(+), 20 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/atlas/blob/5fe6d830/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java
--
diff --git a/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java
b/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java
index 2fe389c..e847014 100644
--- a/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java
+++ b/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java
@@ -153,6 +153,7 @@ public enum AtlasErrorCode {
INVALID_TIMEBOUNDRY_END_TIME(400, "ATLAS-400-00-87C", "Invalid endTime
{0}"),
INVALID_TIMEBOUNDRY_DATERANGE(400, "ATLAS-400-00-87D", "Invalid dateRange:
startTime {0} must be before endTime {1}"),
PROPAGATED_CLASSIFICATION_REMOVAL_NOT_SUPPORTED(400, "ATLAS-400-00-87E",
"Removal of classification {0}, which is propagated from entity {1}, is not
supported"),
+IMPORT_ATTEMPTING_EMPTY_ZIP(400, "ATLAS-400-00-87F", "Attempting to import
empty ZIP file."),
UNAUTHORIZED_ACCESS(403, "ATLAS-403-00-001", "{0} is not authorized to
perform {1}"),
http://git-wip-us.apache.org/repos/asf/atlas/blob/5fe6d830/repository/src/main/java/org/apache/atlas/repository/impexp/ZipSource.java
--
diff --git
a/repository/src/main/java/org/apache/atlas/repository/impexp/ZipSource.java
b/repository/src/main/java/org/apache/atlas/repository/impexp/ZipSource.java
index a292b96..bfa0441 100644
--- a/repository/src/main/java/org/apache/atlas/repository/impexp/ZipSource.java
+++ b/repository/src/main/java/org/apache/atlas/repository/impexp/ZipSource.java
@@ -40,6 +40,8 @@ import java.util.Map;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;
+import static org.apache.atlas.AtlasErrorCode.IMPORT_ATTEMPTING_EMPTY_ZIP;
+
public class ZipSource implements EntityImportStream {
private static final Logger LOG = LoggerFactory.getLogger(ZipSource.class);
@@ -52,16 +54,20 @@ public class ZipSource implements EntityImportStream {
private List entityHandlers;
private int currentPosition;
-public ZipSource(InputStream inputStream) throws IOException {
+public ZipSource(InputStream inputStream) throws IOException,
AtlasBaseException {
this(inputStream, null);
}
-public ZipSource(InputStream inputStream, ImportTransforms
importTransform) throws IOException {
+public ZipSource(InputStream inputStream, ImportTransforms
importTransform) throws IOException, AtlasBaseException {
this.inputStream = inputStream;
this.guidEntityJsonMap = new HashMap<>();
this.importTransform = importTransform;
updateGuidZipEntryMap();
+if (MapUtils.isEmpty(guidEntityJsonMap)) {
+throw new AtlasBaseException(IMPORT_ATTEMPTING_EMPTY_ZIP,
"Attempting to import empty ZIP.");
+}
+
setCreationOrder();
}
@@ -82,7 +88,7 @@ public class ZipSource implements EntityImportStream {
public AtlasTypesDef getTypesDef() throws AtlasBaseException {
final String fileName =
ZipExportFileNames.ATLAS_TYPESDEF_NAME.toString();
-String s = (String) getFromCache(fileName);
+String s = getFromCache(fileName);
return convertFromJson(AtlasTypesDef.class, s);
}
@@ -185,7 +191,12 @@ public class ZipSource implements EntityImportStream {
}
private String getFromCache(String entryName) {
-return guidEntityJsonMap.get(entryName);
+String s = guidEntityJsonMap.get(entryName);
+if (StringUtils.isEmpty(s)) {
+LOG.warn("Could not fetch requested contents of file: {}",
entryName);
+}
+
+return s;
}
public void close() {
@@ -288,6 +299,4 @@ public class ZipSource implements
[02/17] atlas git commit: ATLAS-2875: Implement clear attribute value transformer for Atlas Entity Transformer
ATLAS-2875: Implement clear attribute value transformer for Atlas Entity
Transformer
Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/afa314cb
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/afa314cb
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/afa314cb
Branch: refs/heads/master
Commit: afa314cb520774e6417339f3180d292a6b0c13d6
Parents: 4b3c078
Author: Sarath Subramanian
Authored: Thu Sep 20 11:20:39 2018 -0700
Committer: Ashutosh Mestry
Committed: Thu Oct 11 15:40:21 2018 -0700
--
.../apache/atlas/entitytransform/Action.java| 18
.../entitytransform/BaseEntityHandler.java | 8 ++
.../apache/atlas/entitytransform/Condition.java | 22 +
.../TransformationHandlerTest.java | 99
4 files changed, 147 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/atlas/blob/afa314cb/intg/src/main/java/org/apache/atlas/entitytransform/Action.java
--
diff --git a/intg/src/main/java/org/apache/atlas/entitytransform/Action.java
b/intg/src/main/java/org/apache/atlas/entitytransform/Action.java
index ca5f3a8..f01c6ce 100644
--- a/intg/src/main/java/org/apache/atlas/entitytransform/Action.java
+++ b/intg/src/main/java/org/apache/atlas/entitytransform/Action.java
@@ -31,6 +31,7 @@ public abstract class Action {
private static final String ACTION_NAME_REPLACE_PREFIX = "REPLACE_PREFIX";
private static final String ACTION_NAME_TO_LOWER = "TO_LOWER";
private static final String ACTION_NAME_TO_UPPER = "TO_UPPER";
+private static final String ACTION_NAME_CLEAR = "CLEAR";
protected final String attributeName;
@@ -80,6 +81,10 @@ public abstract class Action {
ret = new SetAction(key, actionValue);
break;
+case ACTION_NAME_CLEAR:
+ret = new ClearAction(key);
+break;
+
default:
ret = new SetAction(key, value); // treat unspecified/unknown
action as 'SET'
break;
@@ -196,4 +201,17 @@ public abstract class Action {
}
}
}
+
+public static class ClearAction extends Action {
+public ClearAction(String attributeName) {
+super(attributeName);
+}
+
+@Override
+public void apply(AtlasTransformableEntity entity) {
+if (isValid() && entity.hasAttribute(attributeName)) {
+entity.setAttribute(attributeName, null);
+}
+}
+}
}
http://git-wip-us.apache.org/repos/asf/atlas/blob/afa314cb/intg/src/main/java/org/apache/atlas/entitytransform/BaseEntityHandler.java
--
diff --git
a/intg/src/main/java/org/apache/atlas/entitytransform/BaseEntityHandler.java
b/intg/src/main/java/org/apache/atlas/entitytransform/BaseEntityHandler.java
index c1f2869..9d44043 100644
--- a/intg/src/main/java/org/apache/atlas/entitytransform/BaseEntityHandler.java
+++ b/intg/src/main/java/org/apache/atlas/entitytransform/BaseEntityHandler.java
@@ -95,6 +95,10 @@ public class BaseEntityHandler {
}
}
+if (CollectionUtils.isEmpty(ret)) {
+ret.add(new BaseEntityHandler(transformers));
+}
+
if (LOG.isDebugEnabled()) {
LOG.debug("<==
BaseEntityHandler.createEntityHandlers(transforms={}): ret.size={}",
transforms, ret.size());
}
@@ -158,6 +162,10 @@ public class BaseEntityHandler {
}
}
+public boolean hasAttribute(String attributeName) {
+return getAttribute(attributeName) != null;
+}
+
public void transformComplete() {
// implementations can override to set value of computed-attributes
}
http://git-wip-us.apache.org/repos/asf/atlas/blob/afa314cb/intg/src/main/java/org/apache/atlas/entitytransform/Condition.java
--
diff --git a/intg/src/main/java/org/apache/atlas/entitytransform/Condition.java
b/intg/src/main/java/org/apache/atlas/entitytransform/Condition.java
index d44f575..bc63079 100644
--- a/intg/src/main/java/org/apache/atlas/entitytransform/Condition.java
+++ b/intg/src/main/java/org/apache/atlas/entitytransform/Condition.java
@@ -31,6 +31,7 @@ public abstract class Condition {
private static final String CONDITION_NAME_EQUALS_IGNORE_CASE =
"EQUALS_IGNORE_CASE";
private static final String CONDITION_NAME_STARTS_WITH =
"STARTS_WITH";
private static final String CONDITION_NAME_STARTS_WITH_IGNORE_CASE =
"STARTS_WITH_IGNORE_CASE";
+private static final String CONDITION_NAME_HAS_VALUE =
[8/8] atlas git commit: ATLAS-2873: Atlas Import Transform Handler Implementation
ATLAS-2873: Atlas Import Transform Handler Implementation
Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/8746b306
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/8746b306
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/8746b306
Branch: refs/heads/master
Commit: 8746b306358637c2a81bb66a8c0d8d82b031286d
Parents: dc6be8e
Author: Sarath Subramanian
Authored: Mon Sep 17 06:18:07 2018 -0700
Committer: Ashutosh Mestry
Committed: Thu Oct 11 15:21:14 2018 -0700
--
.../apache/atlas/entitytransform/Action.java| 199 ++
.../entitytransform/AtlasEntityTransformer.java | 94 +
.../entitytransform/BaseEntityHandler.java | 165 +
.../apache/atlas/entitytransform/Condition.java | 161
.../entitytransform/HdfsPathEntityHandler.java | 170 +
.../HiveColumnEntityHandler.java| 139 +++
.../HiveDatabaseEntityHandler.java | 113 ++
.../HiveStorageDescriptorEntityHandler.java | 143 +++
.../entitytransform/HiveTableEntityHandler.java | 127 +++
.../TransformationConstants.java| 48 +++
.../atlas/model/impexp/AttributeTransform.java | 87 +
.../TransformationHandlerTest.java | 370 +++
.../repository/impexp/ImportTransformer.java| 1 -
13 files changed, 1816 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/atlas/blob/8746b306/intg/src/main/java/org/apache/atlas/entitytransform/Action.java
--
diff --git a/intg/src/main/java/org/apache/atlas/entitytransform/Action.java
b/intg/src/main/java/org/apache/atlas/entitytransform/Action.java
new file mode 100644
index 000..ca5f3a8
--- /dev/null
+++ b/intg/src/main/java/org/apache/atlas/entitytransform/Action.java
@@ -0,0 +1,199 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.entitytransform;
+
+import org.apache.commons.lang.StringUtils;
+import
org.apache.atlas.entitytransform.BaseEntityHandler.AtlasTransformableEntity;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+public abstract class Action {
+private static final Logger LOG = LoggerFactory.getLogger(Action.class);
+
+private static final String ACTION_DELIMITER = ":";
+private static final String ACTION_NAME_SET= "SET";
+private static final String ACTION_NAME_REPLACE_PREFIX = "REPLACE_PREFIX";
+private static final String ACTION_NAME_TO_LOWER = "TO_LOWER";
+private static final String ACTION_NAME_TO_UPPER = "TO_UPPER";
+
+protected final String attributeName;
+
+
+protected Action(String attributeName) {
+this.attributeName = attributeName;
+}
+
+public String getAttributeName() { return attributeName; }
+
+public boolean isValid() {
+return StringUtils.isNotEmpty(attributeName);
+}
+
+public abstract void apply(AtlasTransformableEntity entity);
+
+
+public static Action createAction(String key, String value) {
+if (LOG.isDebugEnabled()) {
+LOG.debug("==> Action.createAction(key={}, value={})", key, value);
+}
+
+final Action ret;
+
+intidxActionDelim = value == null ? -1 :
value.indexOf(ACTION_DELIMITER);
+String actionName = idxActionDelim == -1 ? ACTION_NAME_SET :
value.substring(0, idxActionDelim);
+String actionValue= idxActionDelim == -1 ? value :
value.substring(idxActionDelim + ACTION_DELIMITER.length());
+
+actionName = StringUtils.trim(actionName);
+actionValue = StringUtils.trim(actionValue);
+value = StringUtils.trim(value);
+
+switch (actionName.toUpperCase()) {
+case ACTION_NAME_REPLACE_PREFIX:
+ret = new PrefixReplaceAction(key, actionValue);
+break;
+
+case ACTION_NAME_TO_LOWER:
+ret = new ToLowerCaseAction(key);
+break;
+
+case
[4/8] atlas git commit: ATLAS-2862: Incremental Export now uses request context to determine change marker.
ATLAS-2862: Incremental Export now uses request context to determine change
marker.
Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/8903c9a6
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/8903c9a6
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/8903c9a6
Branch: refs/heads/master
Commit: 8903c9a642001b8aa75e744c741415beb735f241
Parents: 708e486
Author: Ashutosh Mestry
Authored: Mon Sep 10 11:51:49 2018 -0700
Committer: Ashutosh Mestry
Committed: Thu Oct 11 14:25:28 2018 -0700
--
.../atlas/model/impexp/AtlasExportRequest.java | 69 ++
.../atlas/model/impexp/AtlasExportResult.java | 42 +++
.../atlas/model/impexp/AtlasImportResult.java | 2 +-
.../atlas/repository/impexp/AuditsWriter.java | 4 +-
.../atlas/repository/impexp/ExportService.java | 77 +---
.../apache/atlas/repository/impexp/ZipSink.java | 5 +-
.../impexp/ExportIncrementalTest.java | 6 +-
.../impexp/ImportTransformsShaperTest.java | 1 +
.../impexp/ReplicationEntityAttributeTest.java | 2 +-
.../atlas/repository/impexp/ZipSinkTest.java| 2 +-
.../stocksDB-Entities/export-incremental.json | 2 +-
11 files changed, 99 insertions(+), 113 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/atlas/blob/8903c9a6/intg/src/main/java/org/apache/atlas/model/impexp/AtlasExportRequest.java
--
diff --git
a/intg/src/main/java/org/apache/atlas/model/impexp/AtlasExportRequest.java
b/intg/src/main/java/org/apache/atlas/model/impexp/AtlasExportRequest.java
index fc34847..7bb8b76 100644
--- a/intg/src/main/java/org/apache/atlas/model/impexp/AtlasExportRequest.java
+++ b/intg/src/main/java/org/apache/atlas/model/impexp/AtlasExportRequest.java
@@ -23,6 +23,7 @@ import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.databind.annotation.JsonSerialize;
import org.apache.atlas.model.instance.AtlasObjectId;
import org.apache.atlas.model.typedef.AtlasBaseTypeDef;
+import org.apache.commons.collections.MapUtils;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
@@ -46,19 +47,19 @@ public class AtlasExportRequest implements Serializable {
private static final long serialVersionUID = 1L;
-public static final String OPTION_FETCH_TYPE= "fetchType";
-public static final String OPTION_ATTR_MATCH_TYPE = "matchType";
-public static final String OPTION_SKIP_LINEAGE =
"skipLineage";
-public static final String OPTION_KEY_REPLICATED_TO =
"replicatedTo";
-public static final String FETCH_TYPE_FULL = "full";
-public static final String FETCH_TYPE_CONNECTED = "connected";
-public static final String FETCH_TYPE_INCREMENTAL =
"incremental";
-public static final String FETCH_TYPE_INCREMENTAL_FROM_TIME = "fromTime";
-public static final String MATCH_TYPE_STARTS_WITH = "startsWith";
-public static final String MATCH_TYPE_ENDS_WITH = "endsWith";
-public static final String MATCH_TYPE_CONTAINS = "contains";
-public static final String MATCH_TYPE_MATCHES = "matches";
-public static final String MATCH_TYPE_FOR_TYPE = "forType";
+public static final String OPTION_FETCH_TYPE=
"fetchType";
+public static final String OPTION_ATTR_MATCH_TYPE =
"matchType";
+public static final String OPTION_SKIP_LINEAGE =
"skipLineage";
+public static final String OPTION_KEY_REPLICATED_TO =
"replicatedTo";
+public static final String FETCH_TYPE_FULL = "full";
+public static final String FETCH_TYPE_CONNECTED =
"connected";
+public static final String FETCH_TYPE_INCREMENTAL =
"incremental";
+public static final String FETCH_TYPE_INCREMENTAL_CHANGE_MARKER =
"changeMarker";
+public static final String MATCH_TYPE_STARTS_WITH =
"startsWith";
+public static final String MATCH_TYPE_ENDS_WITH =
"endsWith";
+public static final String MATCH_TYPE_CONTAINS =
"contains";
+public static final String MATCH_TYPE_MATCHES =
"matches";
+public static final String MATCH_TYPE_FOR_TYPE =
"forType";
private List itemsToExport = new ArrayList<>();
private Map options = new HashMap<>();
@@ -79,6 +80,48 @@ public class AtlasExportRequest implements Serializable {
this.options = options;
}
+public String getMatchTypeOptionValue() {
+String matchType = null;
+
+
[5/8] atlas git commit: ATLAS-2870: Improvement to AddClassification transform to use filters.
ATLAS-2870: Improvement to AddClassification transform to use filters.
Signed-off-by: Ashutosh Mestry
Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/b08b7b49
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/b08b7b49
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/b08b7b49
Branch: refs/heads/master
Commit: b08b7b49a23db9c60ccb7f1bfe657c5ffdb12f6d
Parents: 8903c9a
Author: Ashutosh Mestry
Authored: Tue Sep 11 17:06:43 2018 -0700
Committer: Ashutosh Mestry
Committed: Thu Oct 11 14:25:39 2018 -0700
--
.../atlas/repository/impexp/ImportService.java | 2 +-
.../repository/impexp/ImportTransformer.java| 58 +++-
.../impexp/ImportTransformsShaper.java | 16 +-
.../repository/impexp/ImportTransformsTest.java | 29 +-
4 files changed, 97 insertions(+), 8 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/atlas/blob/b08b7b49/repository/src/main/java/org/apache/atlas/repository/impexp/ImportService.java
--
diff --git
a/repository/src/main/java/org/apache/atlas/repository/impexp/ImportService.java
b/repository/src/main/java/org/apache/atlas/repository/impexp/ImportService.java
index 095f60f..a88ba2b 100644
---
a/repository/src/main/java/org/apache/atlas/repository/impexp/ImportService.java
+++
b/repository/src/main/java/org/apache/atlas/repository/impexp/ImportService.java
@@ -112,7 +112,7 @@ public class ImportService {
return;
}
-importTransformsShaper.shape(importTransform);
+importTransformsShaper.shape(importTransform,
source.getExportResult().getRequest());
source.setImportTransform(importTransform);
if(LOG.isDebugEnabled()) {
http://git-wip-us.apache.org/repos/asf/atlas/blob/b08b7b49/repository/src/main/java/org/apache/atlas/repository/impexp/ImportTransformer.java
--
diff --git
a/repository/src/main/java/org/apache/atlas/repository/impexp/ImportTransformer.java
b/repository/src/main/java/org/apache/atlas/repository/impexp/ImportTransformer.java
index 70117f6..7bc3536 100644
---
a/repository/src/main/java/org/apache/atlas/repository/impexp/ImportTransformer.java
+++
b/repository/src/main/java/org/apache/atlas/repository/impexp/ImportTransformer.java
@@ -21,10 +21,14 @@ import org.apache.atlas.AtlasErrorCode;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.instance.AtlasClassification;
import org.apache.atlas.model.instance.AtlasEntity;
+import org.apache.atlas.model.instance.AtlasObjectId;
import org.apache.commons.lang.StringUtils;
+import scala.Tuple3;
import java.util.ArrayList;
import java.util.List;
+import java.util.Map;
+import java.util.Objects;
public abstract class ImportTransformer {
@@ -71,8 +75,8 @@ public abstract class ImportTransformer {
} else if (key.equals(TRANSFORMER_SET_DELETED)) {
ret = new SetDeleted();
} else if (key.equals(TRANSFORMER_NAME_ADD_CLASSIFICATION)) {
-String name = (params == null || params.length < 1) ? "" :
StringUtils.join(params, ":", 1, params.length);
-ret = new AddClassification(name);
+String name = (params == null || params.length < 1) ? "" :
params[1];
+ret = new AddClassification(name, (params != null && params.length
== 3) ? params[2] : "");
} else {
throw new AtlasBaseException(AtlasErrorCode.INVALID_VALUE, "Error
creating ImportTransformer. Unknown transformer: {}.", transformerSpec);
}
@@ -151,12 +155,22 @@ public abstract class ImportTransformer {
}
static class AddClassification extends ImportTransformer {
+private static final String FILTER_SCOPE_TOP_LEVEL = "topLevel";
+
+private final String scope;
private final String classificationName;
+private List filters;
-public AddClassification(String name) {
+public AddClassification(String name, String scope) {
super(TRANSFORMER_NAME_REMOVE_CLASSIFICATION);
this.classificationName = name;
+this.scope = scope;
+filters = new ArrayList<>();
+}
+
+public void addFilter(AtlasObjectId objectId) {
+filters.add(objectId);
}
@Override
@@ -166,6 +180,10 @@ public abstract class ImportTransformer {
}
AtlasEntity entity = (AtlasEntity) o;
+if(!passThruFilters(entity)) {
+return o;
+}
+
if(entity.getClassifications() == null) {
entity.setClassifications(new
ArrayList());
}
@@ -180,6 +198,40 @@ public abstract class
[3/8] atlas git commit: ATLAS-2856: added utility methods to RequestContext, to find number of active requests and earliest active request-time
ATLAS-2856: added utility methods to RequestContext, to find number of active
requests and earliest active request-time
Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/708e4865
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/708e4865
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/708e4865
Branch: refs/heads/master
Commit: 708e4865ca51dd11bc8163c81a730773b5f84cb0
Parents: 116fb62
Author: Madhan Neethiraj
Authored: Fri Aug 31 16:04:14 2018 -0700
Committer: Ashutosh Mestry
Committed: Thu Oct 11 14:05:56 2018 -0700
--
.../store/graph/v2/AtlasEntityStoreV2Test.java | 15 -
.../java/org/apache/atlas/RequestContext.java | 33
2 files changed, 47 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/atlas/blob/708e4865/repository/src/test/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2Test.java
--
diff --git
a/repository/src/test/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2Test.java
b/repository/src/test/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2Test.java
index b13a865..4fd2820 100644
---
a/repository/src/test/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2Test.java
+++
b/repository/src/test/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2Test.java
@@ -19,6 +19,7 @@ package org.apache.atlas.repository.store.graph.v2;
import com.google.common.collect.ImmutableSet;
import org.apache.atlas.AtlasErrorCode;
+import org.apache.atlas.RequestContext;
import org.apache.atlas.TestModules;
import org.apache.atlas.TestUtilsV2;
import org.apache.atlas.exception.AtlasBaseException;
@@ -37,15 +38,17 @@ import org.apache.atlas.model.typedef.AtlasEntityDef;
import org.apache.atlas.model.typedef.AtlasTypesDef;
import org.apache.atlas.type.AtlasTypeUtil;
import org.apache.commons.collections.CollectionUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import org.testng.Assert;
import org.testng.annotations.BeforeClass;
+import org.testng.annotations.BeforeTest;
import org.testng.annotations.Guice;
import org.testng.annotations.Test;
import javax.inject.Inject;
import java.util.ArrayList;
import java.util.Arrays;
-import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
@@ -63,6 +66,8 @@ import static org.testng.Assert.fail;
@Guice(modules = TestModules.TestOnlyModule.class)
public class AtlasEntityStoreV2Test extends AtlasEntityTestBase {
+private static final Logger LOG =
LoggerFactory.getLogger(AtlasEntityStoreV2Test.class);
+
private AtlasEntitiesWithExtInfo deptEntity;
private AtlasEntityWithExtInfo dbEntity;
private AtlasEntityWithExtInfo tblEntity;
@@ -102,6 +107,14 @@ public class AtlasEntityStoreV2Test extends
AtlasEntityTestBase {
typeDefStore.createTypesDef(typesDef11);
}
+@BeforeTest
+public void init() throws Exception {
+entityStore = new AtlasEntityStoreV2(deleteHandler, typeRegistry,
mockChangeNotifier, graphMapper);
+RequestContext.clear();
+RequestContext.get().setUser(TestUtilsV2.TEST_USER, null);
+
+LOG.debug("RequestContext: activeCount={},
earliestActiveRequestTime={}", RequestContext.getActiveRequestsCount(),
RequestContext.earliestActiveRequestTime());
+}
@Test
public void testDefaultValueForPrimitiveTypes() throws Exception {
http://git-wip-us.apache.org/repos/asf/atlas/blob/708e4865/server-api/src/main/java/org/apache/atlas/RequestContext.java
--
diff --git a/server-api/src/main/java/org/apache/atlas/RequestContext.java
b/server-api/src/main/java/org/apache/atlas/RequestContext.java
index 8fe10fe..25a35ce 100644
--- a/server-api/src/main/java/org/apache/atlas/RequestContext.java
+++ b/server-api/src/main/java/org/apache/atlas/RequestContext.java
@@ -32,6 +32,7 @@ public class RequestContext {
private static final Logger LOG =
LoggerFactory.getLogger(RequestContext.class);
private static final ThreadLocal CURRENT_CONTEXT = new
ThreadLocal<>();
+private static final Set ACTIVE_REQUESTS = new
HashSet<>();
private final Map updatedEntities =
new HashMap<>();
private final Map deletedEntities =
new HashMap<>();
@@ -60,6 +61,10 @@ public class RequestContext {
if (ret == null) {
ret = new RequestContext();
CURRENT_CONTEXT.set(ret);
+
+synchronized (ACTIVE_REQUESTS) {
+ACTIVE_REQUESTS.add(ret);
+}
}
return ret;
@@ -79,6 +84,10 @@ public class
[03/17] atlas git commit: ATLAS-2882: AddClassification transform for new transforms
ATLAS-2882: AddClassification transform for new transforms
Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/9d4f9728
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/9d4f9728
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/9d4f9728
Branch: refs/heads/master
Commit: 9d4f972892cc4248e4e1b0cb5f2933806a4e0fbc
Parents: afa314c
Author: Ashutosh Mestry
Authored: Thu Sep 20 12:54:36 2018 -0700
Committer: Ashutosh Mestry
Committed: Thu Oct 11 15:40:34 2018 -0700
--
.../apache/atlas/entitytransform/Action.java| 68 +++
.../entitytransform/AtlasEntityTransformer.java | 11 +-
.../entitytransform/BaseEntityHandler.java | 101 ++--
.../apache/atlas/entitytransform/Condition.java | 86 +
.../atlas/entitytransform/NeedsContext.java | 23
.../entitytransform/TransformerContext.java | 47
.../TransformationHandlerTest.java | 120 +--
.../atlas/repository/impexp/ImportService.java | 36 ++
.../atlas/repository/impexp/ZipSource.java | 6 +-
9 files changed, 424 insertions(+), 74 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/atlas/blob/9d4f9728/intg/src/main/java/org/apache/atlas/entitytransform/Action.java
--
diff --git a/intg/src/main/java/org/apache/atlas/entitytransform/Action.java
b/intg/src/main/java/org/apache/atlas/entitytransform/Action.java
index f01c6ce..fa18558 100644
--- a/intg/src/main/java/org/apache/atlas/entitytransform/Action.java
+++ b/intg/src/main/java/org/apache/atlas/entitytransform/Action.java
@@ -17,16 +17,26 @@
*/
package org.apache.atlas.entitytransform;
+import org.apache.atlas.exception.AtlasBaseException;
+import org.apache.atlas.model.instance.AtlasClassification;
+import org.apache.atlas.model.instance.AtlasEntity;
+import org.apache.atlas.model.typedef.AtlasClassificationDef;
+import org.apache.atlas.model.typedef.AtlasTypesDef;
import org.apache.commons.lang.StringUtils;
import
org.apache.atlas.entitytransform.BaseEntityHandler.AtlasTransformableEntity;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.util.ArrayList;
+import java.util.Collections;
+
public abstract class Action {
private static final Logger LOG = LoggerFactory.getLogger(Action.class);
+private static final String ENTITY_KEY = "__entity";
private static final String ACTION_DELIMITER = ":";
+private static final String ACTION_ADD_CLASSIFICATION =
"ADDCLASSIFICATION";
private static final String ACTION_NAME_SET= "SET";
private static final String ACTION_NAME_REPLACE_PREFIX = "REPLACE_PREFIX";
private static final String ACTION_NAME_TO_LOWER = "TO_LOWER";
@@ -65,6 +75,10 @@ public abstract class Action {
value = StringUtils.trim(value);
switch (actionName.toUpperCase()) {
+case ACTION_ADD_CLASSIFICATION:
+ret = new AddClassificationAction(actionValue);
+break;
+
case ACTION_NAME_REPLACE_PREFIX:
ret = new PrefixReplaceAction(key, actionValue);
break;
@@ -115,6 +129,60 @@ public abstract class Action {
}
}
+public static class AddClassificationAction extends Action implements
NeedsContext {
+
+private final String classificationName;
+private TransformerContext transformerContext;
+
+public AddClassificationAction(String classificationName) {
+super(ENTITY_KEY);
+
+this.classificationName = classificationName;
+}
+
+@Override
+public void apply(AtlasTransformableEntity transformableEntity) {
+AtlasEntity entity = transformableEntity.entity;
+if (entity.getClassifications() == null) {
+entity.setClassifications(new
ArrayList());
+}
+
+for (AtlasClassification c : entity.getClassifications()) {
+if (c.getTypeName().equals(classificationName)) {
+return;
+}
+}
+
+entity.getClassifications().add(new
AtlasClassification(classificationName));
+}
+
+@Override
+public void setContext(TransformerContext transformerContext) {
+this.transformerContext = transformerContext;
+getCreateTag(classificationName);
+}
+
+private void getCreateTag(String classificationName) {
+if (transformerContext == null) {
+return;
+}
+
+try {
+AtlasClassificationDef classificationDef =
[07/17] atlas git commit: ATLAS-2892: Delete by name REST endpoint.
ATLAS-2892: Delete by name REST endpoint.
Change-Id: I9b0a40b42bc945f51aa098e1b15f28f7d2c9fee2
Signed-off-by: Ashutosh Mestry
Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/f4dac184
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/f4dac184
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/f4dac184
Branch: refs/heads/master
Commit: f4dac18468b94f56fbe482853982d064fc561cd2
Parents: b04c5bd
Author: Ashutosh Mestry
Authored: Wed Sep 26 13:02:53 2018 -0700
Committer: Ashutosh Mestry
Committed: Thu Oct 11 17:21:26 2018 -0700
--
dashboardv2/public/js/models/VTag.js| 6 +--
.../public/js/views/tag/TagLayoutView.js| 2 +-
.../apache/atlas/store/AtlasTypeDefStore.java | 3 ++
.../store/graph/AtlasTypeDefGraphStore.java | 26 +
.../store/graph/AtlasTypeDefGraphStoreTest.java | 21
.../src/test/resources/json/hiveDBv2.json | 56
.../org/apache/atlas/web/rest/TypesREST.java| 26 +
7 files changed, 134 insertions(+), 6 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/atlas/blob/f4dac184/dashboardv2/public/js/models/VTag.js
--
diff --git a/dashboardv2/public/js/models/VTag.js
b/dashboardv2/public/js/models/VTag.js
index 384bcc3..d436bb5 100644
--- a/dashboardv2/public/js/models/VTag.js
+++ b/dashboardv2/public/js/models/VTag.js
@@ -49,11 +49,7 @@ define(['require',
return this.constructor.nonCrudOperation.call(this, url, 'DELETE',
options);
},
deleteTag: function(options) {
-var url = UrlLinks.classificationDefApiUrl();
-options = _.extend({
-contentType: 'application/json',
-dataType: 'json'
-}, options);
+var url = UrlLinks.classificationDefApiUrl(options.typeName);
return this.constructor.nonCrudOperation.call(this, url, 'DELETE',
options);
},
saveTagAttribute: function(options) {
http://git-wip-us.apache.org/repos/asf/atlas/blob/f4dac184/dashboardv2/public/js/views/tag/TagLayoutView.js
--
diff --git a/dashboardv2/public/js/views/tag/TagLayoutView.js
b/dashboardv2/public/js/views/tag/TagLayoutView.js
index 7381e64..a5df515 100644
--- a/dashboardv2/public/js/views/tag/TagLayoutView.js
+++ b/dashboardv2/public/js/views/tag/TagLayoutView.js
@@ -578,7 +578,7 @@ define(['require',
structDefs: []
};
deleteTagData.deleteTag({
-data: JSON.stringify(deleteJson),
+typeName: that.tag,
success: function() {
Utils.notifySuccess({
content: "Classification " + that.tag +
Messages.deleteSuccessMessage
http://git-wip-us.apache.org/repos/asf/atlas/blob/f4dac184/intg/src/main/java/org/apache/atlas/store/AtlasTypeDefStore.java
--
diff --git a/intg/src/main/java/org/apache/atlas/store/AtlasTypeDefStore.java
b/intg/src/main/java/org/apache/atlas/store/AtlasTypeDefStore.java
index 025f89a..9a74627 100644
--- a/intg/src/main/java/org/apache/atlas/store/AtlasTypeDefStore.java
+++ b/intg/src/main/java/org/apache/atlas/store/AtlasTypeDefStore.java
@@ -99,8 +99,11 @@ public interface AtlasTypeDefStore {
AtlasTypesDef searchTypesDef(SearchFilter searchFilter) throws
AtlasBaseException;
+
/* Generic operation */
AtlasBaseTypeDef getByName(String name) throws AtlasBaseException;
AtlasBaseTypeDef getByGuid(String guid) throws AtlasBaseException;
+
+void deleteTypeByName(String typeName) throws AtlasBaseException;
}
http://git-wip-us.apache.org/repos/asf/atlas/blob/f4dac184/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasTypeDefGraphStore.java
--
diff --git
a/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasTypeDefGraphStore.java
b/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasTypeDefGraphStore.java
index bd82eb5..b142179 100644
---
a/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasTypeDefGraphStore.java
+++
b/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasTypeDefGraphStore.java
@@ -40,6 +40,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
@@ -598,6 +599,31 @@ public abstract class AtlasTypeDefGraphStore implements
AtlasTypeDefStore {
}
[04/17] atlas git commit: ATLAS-2888: Export & Import Process: Change Marker Removed Inadvertently
ATLAS-2888: Export & Import Process: Change Marker Removed Inadvertently
Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/b04c5bd2
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/b04c5bd2
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/b04c5bd2
Branch: refs/heads/master
Commit: b04c5bd28c9a1685d42bd2294478ab0f350ac608
Parents: 8639ada
Author: Ashutosh Mestry
Authored: Sun Sep 23 22:51:08 2018 -0700
Committer: Ashutosh Mestry
Committed: Thu Oct 11 17:21:25 2018 -0700
--
.../atlas/repository/impexp/AuditsWriter.java | 38 ++--
1 file changed, 10 insertions(+), 28 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/atlas/blob/b04c5bd2/repository/src/main/java/org/apache/atlas/repository/impexp/AuditsWriter.java
--
diff --git
a/repository/src/main/java/org/apache/atlas/repository/impexp/AuditsWriter.java
b/repository/src/main/java/org/apache/atlas/repository/impexp/AuditsWriter.java
index f72de56..cc10660 100644
---
a/repository/src/main/java/org/apache/atlas/repository/impexp/AuditsWriter.java
+++
b/repository/src/main/java/org/apache/atlas/repository/impexp/AuditsWriter.java
@@ -130,6 +130,10 @@ public class AuditsWriter {
return StringUtils.split(fullName, "$")[1];
}
+private void saveCurrentServer() throws AtlasBaseException {
+saveServer(getCurrentClusterName(), getCurrentClusterName());
+}
+
private class ExportAudits {
private AtlasExportRequest request;
private String targetServerName;
@@ -144,8 +148,10 @@ public class AuditsWriter {
request = result.getRequest();
replicationOptionState =
isReplicationOptionSet(request.getOptions(), optionKeyReplicatedTo);
-saveServers();
+saveCurrentServer();
+targetServerFullName =
getClusterNameFromOptions(request.getOptions(), optionKeyReplicatedTo);
+targetServerName = getServerNameFromFullName(targetServerFullName);
auditService.add(userName, getCurrentClusterName(),
targetServerName,
ExportImportAuditEntry.OPERATION_EXPORT,
AtlasType.toJson(result), startTime, endTime,
!entityGuids.isEmpty());
@@ -157,16 +163,6 @@ public class AuditsWriter {
updateReplicationAttribute(replicationOptionState,
targetServerName, targetServerFullName,
entityGuids, Constants.ATTR_NAME_REPLICATED_TO,
result.getChangeMarker());
}
-
-private void saveServers() throws AtlasBaseException {
-saveServer(getCurrentClusterName(), getCurrentClusterName());
-
-targetServerFullName =
getClusterNameFromOptions(request.getOptions(), optionKeyReplicatedTo);
-targetServerName = getServerNameFromFullName(targetServerFullName);
-if(StringUtils.isNotEmpty(targetServerName)) {
-saveServer(targetServerName, targetServerFullName);
-}
-}
}
private class ImportAudits {
@@ -183,8 +179,10 @@ public class AuditsWriter {
request = result.getRequest();
replicationOptionState =
isReplicationOptionSet(request.getOptions(), optionKeyReplicatedFrom);
-saveServers();
+saveCurrentServer();
+sourceServerFullName =
getClusterNameFromOptions(request.getOptions(), optionKeyReplicatedFrom);
+sourceServerName = getServerNameFromFullName(sourceServerFullName);
auditService.add(userName,
sourceServerName, getCurrentClusterName(),
ExportImportAuditEntry.OPERATION_IMPORT,
@@ -197,21 +195,5 @@ public class AuditsWriter {
updateReplicationAttribute(replicationOptionState,
sourceServerName, sourceServerFullName, entityGuids,
Constants.ATTR_NAME_REPLICATED_FROM,
result.getExportResult().getChangeMarker());
}
-
-private void saveServers() throws AtlasBaseException {
-saveServer(getCurrentClusterName(), getCurrentClusterName());
-
-sourceServerFullName = getClusterNameFromOptionsState();
-sourceServerName = getServerNameFromFullName(sourceServerFullName);
-if(StringUtils.isNotEmpty(sourceServerName)) {
-saveServer(sourceServerName, sourceServerFullName);
-}
-}
-
-private String getClusterNameFromOptionsState() {
-return replicationOptionState
-? getClusterNameFromOptions(request.getOptions(),
optionKeyReplicatedFrom)
-: StringUtils.EMPTY;
-}
}
}
atlas git commit: ATLAS-2814: Fixed merge error.
Repository: atlas
Updated Branches:
refs/heads/master 1eb995434 -> 614415e0a
ATLAS-2814: Fixed merge error.
Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/614415e0
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/614415e0
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/614415e0
Branch: refs/heads/master
Commit: 614415e0a7941e70cc4211ed2213b19057f02ec7
Parents: 1eb9954
Author: Ashutosh Mestry
Authored: Thu Oct 11 22:36:01 2018 -0700
Committer: Ashutosh Mestry
Committed: Thu Oct 11 22:36:01 2018 -0700
--
.../patches/001-base_model_replication_attributes.json | 10 --
.../atlas/repository/impexp/AtlasServerService.java | 11 ---
2 files changed, 16 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/atlas/blob/614415e0/addons/models/-Area0/patches/001-base_model_replication_attributes.json
--
diff --git
a/addons/models/-Area0/patches/001-base_model_replication_attributes.json
b/addons/models/-Area0/patches/001-base_model_replication_attributes.json
index 323a818..92c561f 100644
---
a/addons/models/-Area0/patches/001-base_model_replication_attributes.json
+++
b/addons/models/-Area0/patches/001-base_model_replication_attributes.json
@@ -13,7 +13,10 @@
"cardinality": "SET",
"isIndexable": false,
"isOptional": true,
- "isUnique": false
+ "isUnique": false,
+ "options": {
+"isSoftReference": "true"
+ }
},
{
"name": "replicatedTo",
@@ -21,7 +24,10 @@
"cardinality": "SET",
"isIndexable": false,
"isOptional": true,
- "isUnique": false
+ "isUnique": false,
+ "options": {
+"isSoftReference": "true"
+ }
}
]
}
http://git-wip-us.apache.org/repos/asf/atlas/blob/614415e0/repository/src/main/java/org/apache/atlas/repository/impexp/AtlasServerService.java
--
diff --git
a/repository/src/main/java/org/apache/atlas/repository/impexp/AtlasServerService.java
b/repository/src/main/java/org/apache/atlas/repository/impexp/AtlasServerService.java
index 8859a9d..3e0d580 100644
---
a/repository/src/main/java/org/apache/atlas/repository/impexp/AtlasServerService.java
+++
b/repository/src/main/java/org/apache/atlas/repository/impexp/AtlasServerService.java
@@ -88,8 +88,13 @@ public class AtlasServerService {
}
@GraphTransaction
-public AtlasServer save(AtlasServer server) throws AtlasBaseException {
- return dataAccess.save(server);
+public AtlasServer save(AtlasServer server) {
+try {
+return dataAccess.save(server);
+}
+catch (AtlasBaseException ex) {
+return server;
+}
}
@GraphTransaction
@@ -100,7 +105,7 @@ public class AtlasServerService {
AtlasObjectId objectId = getObjectId(server);
for (String guid : entityGuids) {
-AtlasEntity.AtlasEntityWithExtInfo entityWithExtInfo =
entityStore.getById(guid);
+AtlasEntity.AtlasEntityWithExtInfo entityWithExtInfo =
entityStore.getById(guid, false);
updateAttribute(entityWithExtInfo, attributeName, objectId);
}
}
atlas git commit: ATLAS-2918: Update JanusGraph version to 0.3.1
Repository: atlas
Updated Branches:
refs/heads/master 614415e0a -> ab801ff27
ATLAS-2918: Update JanusGraph version to 0.3.1
Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/ab801ff2
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/ab801ff2
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/ab801ff2
Branch: refs/heads/master
Commit: ab801ff27101c80a553e58a1209b14b492b7435f
Parents: 614415e
Author: Sarath Subramanian
Authored: Thu Oct 11 22:55:35 2018 -0700
Committer: Sarath Subramanian
Committed: Thu Oct 11 22:55:35 2018 -0700
--
graphdb/janus/pom.xml| 1 -
intg/src/test/resources/atlas-application.properties | 1 +
pom.xml | 1 +
3 files changed, 2 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/atlas/blob/ab801ff2/graphdb/janus/pom.xml
--
diff --git a/graphdb/janus/pom.xml b/graphdb/janus/pom.xml
index cd5c82e..5d491e8 100644
--- a/graphdb/janus/pom.xml
+++ b/graphdb/janus/pom.xml
@@ -35,7 +35,6 @@
3.3.3
-0.3.0
false
7.3.0
http://git-wip-us.apache.org/repos/asf/atlas/blob/ab801ff2/intg/src/test/resources/atlas-application.properties
--
diff --git a/intg/src/test/resources/atlas-application.properties
b/intg/src/test/resources/atlas-application.properties
index 7e74d51..159f98b 100644
--- a/intg/src/test/resources/atlas-application.properties
+++ b/intg/src/test/resources/atlas-application.properties
@@ -45,6 +45,7 @@ atlas.graphdb.backend=${graphdb.backend.impl}
# Graph Storage
atlas.graph.storage.backend=${graph.storage.backend}
+atlas.graph.storage.transactions=false
# Entity repository implementation
atlas.EntityAuditRepository.impl=${entity.repository.impl}
http://git-wip-us.apache.org/repos/asf/atlas/blob/ab801ff2/pom.xml
--
diff --git a/pom.xml b/pom.xml
index 92b3218..91ec2c8 100644
--- a/pom.xml
+++ b/pom.xml
@@ -656,6 +656,7 @@
2.11
2.11.0
3.4.6
+0.3.1
3.2.11
1.2.17
