This is an automated email from the ASF dual-hosted git repository.
sarathsubramanian pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/atlas.git
The following commit(s) were added to refs/heads/master by this push:
new d96d3c6c6 ATLAS-5337: Fix Trino Extractor Jersey client failures for
AtlasEntityWithExtInfo POST/GET (#690)
d96d3c6c6 is described below
commit d96d3c6c6a53e20f2f32c507ff0198bf356f48cb
Author: Ramachandran Krishnan <[email protected]>
AuthorDate: Mon Jul 6 20:16:57 2026 +0530
ATLAS-5337: Fix Trino Extractor Jersey client failures for
AtlasEntityWithExtInfo POST/GET (#690)
* ATLAS-5337: Fix Trino Extractor Jersey client failures for
AtlasEntityWithExtInfo
Remove jersey-client 1.9 pin from trino-extractor; JSON-serialize entity
POSTs
via AtlasType.toJson() in AtlasClientV2; parse model GET responses with
AtlasJson in AtlasBaseClient; support ATLAS_USERNAME/ATLAS_PASSWORD env vars
in AuthenticationUtil for non-interactive runs.
* Fix flaky QuickStart IT assertions for non-deterministic list order.
QuickStartIT and QuickStartV2IT assumed fixed column/process input ordering
from graph traversal, which fails intermittently on Java 17 CI.
* CI: retrigger workflow after GlossaryServiceTest flake.
* CI: retrigger workflow after atlas-hbase docker flake.
* Fix CI flakes in GlossaryServiceTest, BasicSearchIT, and atlas-hbase
docker.
Retry transient JanusGraph errors during glossary test setup, scope
BasicSearchIT
hive queries to the imported @cl1 dataset, and harden atlas-hbase container
startup
and health checks so docker compose --wait does not fail spuriously.
---------
Co-authored-by: ramk <[email protected]>
---
.github/workflows/ci.yml | 3 +
addons/trino-extractor/pom.xml | 5 --
.../main/java/org/apache/atlas/AtlasClientV2.java | 10 +--
.../java/org/apache/atlas/AtlasBaseClient.java | 12 ++++
.../atlas-docker/docker-compose.atlas-hbase.yml | 3 +-
dev-support/atlas-docker/scripts/atlas-hbase.sh | 16 ++++-
.../org/apache/atlas/utils/AuthenticationUtil.java | 6 ++
.../apache/atlas/glossary/GlossaryServiceTest.java | 75 +++++++++++++++++++---
.../org/apache/atlas/examples/QuickStartIT.java | 19 ++++--
.../org/apache/atlas/examples/QuickStartV2IT.java | 12 +++-
.../atlas/web/integration/BasicSearchIT.java | 43 +++++++++++--
11 files changed, 170 insertions(+), 34 deletions(-)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 721c039c3..ededed192 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -145,6 +145,9 @@ jobs:
echo "All required containers are up and running";
docker stop $(docker ps -q) && docker rm $(docker ps -aq);
else
+ for container in "${containers[@]}"; do
+ docker logs --tail 200 "$container" 2>&1 || true
+ done
docker stop $(docker ps -q) && docker rm $(docker ps -aq);
exit 1;
fi
diff --git a/addons/trino-extractor/pom.xml b/addons/trino-extractor/pom.xml
index a7a66b937..b66b957ca 100644
--- a/addons/trino-extractor/pom.xml
+++ b/addons/trino-extractor/pom.xml
@@ -42,11 +42,6 @@
<groupId>ch.qos.logback</groupId>
<artifactId>logback-classic</artifactId>
</dependency>
- <dependency>
- <groupId>com.sun.jersey</groupId>
- <artifactId>jersey-client</artifactId>
- <version>1.9</version>
- </dependency>
<dependency>
<groupId>io.trino</groupId>
diff --git a/client/client-v2/src/main/java/org/apache/atlas/AtlasClientV2.java
b/client/client-v2/src/main/java/org/apache/atlas/AtlasClientV2.java
index a46a6724a..daa600949 100644
--- a/client/client-v2/src/main/java/org/apache/atlas/AtlasClientV2.java
+++ b/client/client-v2/src/main/java/org/apache/atlas/AtlasClientV2.java
@@ -457,25 +457,25 @@ public class AtlasClientV2 extends AtlasBaseClient {
}
public EntityMutationResponse createEntity(AtlasEntityWithExtInfo entity)
throws AtlasServiceException {
- return callAPI(API_V2.CREATE_ENTITY, EntityMutationResponse.class,
entity);
+ return callAPI(API_V2.CREATE_ENTITY, EntityMutationResponse.class,
AtlasType.toJson(entity));
}
public EntityMutationResponse createEntities(AtlasEntitiesWithExtInfo
atlasEntities) throws AtlasServiceException {
- return callAPI(API_V2.CREATE_ENTITIES, EntityMutationResponse.class,
atlasEntities);
+ return callAPI(API_V2.CREATE_ENTITIES, EntityMutationResponse.class,
AtlasType.toJson(atlasEntities));
}
public EntityMutationResponse updateEntity(AtlasEntityWithExtInfo entity)
throws AtlasServiceException {
- return callAPI(API_V2.UPDATE_ENTITY, EntityMutationResponse.class,
entity);
+ return callAPI(API_V2.UPDATE_ENTITY, EntityMutationResponse.class,
AtlasType.toJson(entity));
}
public EntityMutationResponse updateEntities(AtlasEntitiesWithExtInfo
atlasEntities) throws AtlasServiceException {
- return callAPI(API_V2.UPDATE_ENTITIES, EntityMutationResponse.class,
atlasEntities);
+ return callAPI(API_V2.UPDATE_ENTITIES, EntityMutationResponse.class,
AtlasType.toJson(atlasEntities));
}
public EntityMutationResponse updateEntityByAttribute(String typeName,
Map<String, String> uniqAttributes, AtlasEntityWithExtInfo entityInfo) throws
AtlasServiceException {
MultivaluedMap<String, String> queryParams =
attributesToQueryParams(uniqAttributes);
- return callAPI(API_V2.UPDATE_ENTITY_BY_ATTRIBUTE,
EntityMutationResponse.class, entityInfo, queryParams, typeName);
+ return callAPI(API_V2.UPDATE_ENTITY_BY_ATTRIBUTE,
EntityMutationResponse.class, AtlasType.toJson(entityInfo), queryParams,
typeName);
}
public EntityMutationResponse partialUpdateEntityByGuid(String entityGuid,
Object attrValue, String attrName) throws AtlasServiceException {
diff --git a/client/common/src/main/java/org/apache/atlas/AtlasBaseClient.java
b/client/common/src/main/java/org/apache/atlas/AtlasBaseClient.java
index b6371d32f..9b62e82d9 100644
--- a/client/common/src/main/java/org/apache/atlas/AtlasBaseClient.java
+++ b/client/common/src/main/java/org/apache/atlas/AtlasBaseClient.java
@@ -496,6 +496,14 @@ public abstract class AtlasBaseClient {
} catch (IOException e) {
throw new AtlasServiceException(api, e);
}
+ } else if (isAtlasModelType(responseType.getRawClass())) {
+ String stringEntity =
clientResponse.getEntity(String.class);
+ T entity = AtlasJson.fromJson(stringEntity,
responseType.getRawClass());
+
+ LOG.debug("Response : {}", entity);
+
LOG.debug("------------------------------------------------------");
+
+ return entity;
} else {
T entity = clientResponse.getEntity(responseType);
@@ -657,6 +665,10 @@ public abstract class AtlasBaseClient {
this.configuration = configuration;
}
+ private static boolean isAtlasModelType(Class<?> clazz) {
+ return clazz != null &&
clazz.getName().startsWith("org.apache.atlas.model.");
+ }
+
@VisibleForTesting
void setService(WebResource resource) {
this.service = resource;
diff --git a/dev-support/atlas-docker/docker-compose.atlas-hbase.yml
b/dev-support/atlas-docker/docker-compose.atlas-hbase.yml
index 0e2d5f9da..9d46702c2 100644
--- a/dev-support/atlas-docker/docker-compose.atlas-hbase.yml
+++ b/dev-support/atlas-docker/docker-compose.atlas-hbase.yml
@@ -25,8 +25,9 @@ services:
condition: service_started
atlas-kafka:
condition: service_started
+ restart: unless-stopped
healthcheck:
- test: [ "CMD", "wget", "-q", "--spider",
"http://localhost:16030/rs-status"; ]
+ test: [ "CMD-SHELL", "wget -q --spider http://localhost:16030/rs-status
&& wget -q --spider http://localhost:16010/master-status"; ]
interval: 30s
timeout: 10s
retries: 30
diff --git a/dev-support/atlas-docker/scripts/atlas-hbase.sh
b/dev-support/atlas-docker/scripts/atlas-hbase.sh
index 81b382b5d..e139a9377 100755
--- a/dev-support/atlas-docker/scripts/atlas-hbase.sh
+++ b/dev-support/atlas-docker/scripts/atlas-hbase.sh
@@ -33,7 +33,21 @@ fi
su -c "${HBASE_HOME}/bin/start-hbase.sh" hbase
-HBASE_MASTER_PID=`ps -ef | grep -v grep | grep -i
"org.apache.hadoop.hbase.master.HMaster" | awk '{print $2}'`
+HBASE_MASTER_PID=""
+for attempt in $(seq 1 60); do
+ HBASE_MASTER_PID=`ps -ef | grep -v grep | grep -i
"org.apache.hadoop.hbase.master.HMaster" | awk '{print $2}'`
+
+ if [ -n "${HBASE_MASTER_PID}" ]; then
+ break
+ fi
+
+ sleep 2
+done
+
+if [ -z "${HBASE_MASTER_PID}" ]; then
+ echo "HBase HMaster failed to start" >&2
+ exit 1
+fi
# prevent the container from exiting
tail --pid=$HBASE_MASTER_PID -f /dev/null
diff --git a/intg/src/main/java/org/apache/atlas/utils/AuthenticationUtil.java
b/intg/src/main/java/org/apache/atlas/utils/AuthenticationUtil.java
index d98de2812..229f17786 100644
--- a/intg/src/main/java/org/apache/atlas/utils/AuthenticationUtil.java
+++ b/intg/src/main/java/org/apache/atlas/utils/AuthenticationUtil.java
@@ -74,6 +74,12 @@ public final class AuthenticationUtil {
}
public static String[] getBasicAuthenticationInput() {
+ String envUser = System.getenv("ATLAS_USERNAME");
+ String envPass = System.getenv("ATLAS_PASSWORD");
+ if (envUser != null && !envUser.isEmpty() && envPass != null) {
+ return new String[] {envUser, envPass};
+ }
+
String username = null;
String password = null;
diff --git
a/repository/src/test/java/org/apache/atlas/glossary/GlossaryServiceTest.java
b/repository/src/test/java/org/apache/atlas/glossary/GlossaryServiceTest.java
index be32b0344..f8cac1c3c 100644
---
a/repository/src/test/java/org/apache/atlas/glossary/GlossaryServiceTest.java
+++
b/repository/src/test/java/org/apache/atlas/glossary/GlossaryServiceTest.java
@@ -152,20 +152,15 @@ public class GlossaryServiceTest {
@BeforeClass
public void setupSampleGlossary() {
try {
- TestLoadModelUtils.loadAllModels("0000-Area0", typeDefStore,
typeRegistry);
+ loadAllModelsWithRetry("0000-Area0");
} catch (AtlasBaseException | IOException e) {
- throw new SkipException("SubjectArea model loading failed");
+ throw new SkipException("SubjectArea model loading failed: " +
e.getMessage());
}
try {
- AtlasClassificationDef classificationDef = new
AtlasClassificationDef("TestClassification", "Test only classification");
- AtlasTypesDef typesDef = new AtlasTypesDef();
-
-
typesDef.setClassificationDefs(Collections.singletonList(classificationDef));
-
- typeDefStore.createTypesDef(typesDef);
+ createTestClassificationWithRetry();
} catch (AtlasBaseException e) {
- throw new SkipException("Test classification creation failed");
+ throw new SkipException("Test classification creation failed: " +
e.getMessage());
}
// Glossary
@@ -1416,6 +1411,68 @@ public class GlossaryServiceTest {
}
}
+ private void loadAllModelsWithRetry(String dirName) throws
AtlasBaseException, IOException {
+ int maxAttempts = 5;
+
+ for (int attempt = 1; attempt <= maxAttempts; attempt++) {
+ try {
+ TestLoadModelUtils.loadAllModels(dirName, typeDefStore,
typeRegistry);
+ return;
+ } catch (AtlasBaseException e) {
+ if (attempt == maxAttempts || !isTransientGraphError(e)) {
+ throw e;
+ }
+
+ LOG.warn("loadAllModels failed (attempt {}/{}), retrying: {}",
attempt, maxAttempts, e.getMessage());
+ sleepBeforeRetry(attempt);
+ }
+ }
+ }
+
+ private void createTestClassificationWithRetry() throws AtlasBaseException
{
+ AtlasClassificationDef classificationDef = new
AtlasClassificationDef("TestClassification", "Test only classification");
+ AtlasTypesDef typesDef = new AtlasTypesDef();
+
+
typesDef.setClassificationDefs(Collections.singletonList(classificationDef));
+
+ int maxAttempts = 5;
+
+ for (int attempt = 1; attempt <= maxAttempts; attempt++) {
+ try {
+ typeDefStore.createTypesDef(typesDef);
+ return;
+ } catch (AtlasBaseException e) {
+ if (attempt == maxAttempts || !isTransientGraphError(e)) {
+ throw e;
+ }
+
+ LOG.warn("createTypesDef failed (attempt {}/{}), retrying:
{}", attempt, maxAttempts, e.getMessage());
+ sleepBeforeRetry(attempt);
+ }
+ }
+ }
+
+ private static boolean isTransientGraphError(AtlasBaseException e) {
+ if (e == null || e.getMessage() == null) {
+ return false;
+ }
+
+ String message = e.getMessage().toLowerCase();
+
+ return message.contains("could not start new transaction")
+ || message.contains("cursor has been closed")
+ || message.contains("permanentlockingexception");
+ }
+
+ private static void sleepBeforeRetry(int attempt) throws
AtlasBaseException {
+ try {
+ Thread.sleep(500L * attempt);
+ } catch (InterruptedException ie) {
+ Thread.currentThread().interrupt();
+ throw new AtlasBaseException(AtlasErrorCode.INTERNAL_ERROR, ie);
+ }
+ }
+
private static InputStream getFile(String subDir, String fileName) {
final String userDir = System.getProperty("user.dir");
String filePath = getTestFilePath(userDir, subDir, fileName);
diff --git a/webapp/src/test/java/org/apache/atlas/examples/QuickStartIT.java
b/webapp/src/test/java/org/apache/atlas/examples/QuickStartIT.java
index 707add619..1c0b2fb6c 100644
--- a/webapp/src/test/java/org/apache/atlas/examples/QuickStartIT.java
+++ b/webapp/src/test/java/org/apache/atlas/examples/QuickStartIT.java
@@ -30,6 +30,8 @@ import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertNotNull;
@@ -80,9 +82,11 @@ public class QuickStartIT extends BaseResourceIT {
String timeDimTableId = getTableId(QuickStart.TIME_DIM_TABLE);
String salesFactDailyMVId =
getTableId(QuickStart.SALES_FACT_DAILY_MV_TABLE);
- assertEquals(salesFactTableId, inputs.get(0)._getId());
- assertEquals(timeDimTableId, inputs.get(1)._getId());
- assertEquals(salesFactDailyMVId, outputs.get(0)._getId());
+ Set<String> inputIds =
inputs.stream().map(Id::_getId).collect(Collectors.toSet());
+
+ assertTrue(inputIds.contains(salesFactTableId));
+ assertTrue(inputIds.contains(timeDimTableId));
+ assertEquals(outputs.get(0)._getId(), salesFactDailyMVId);
}
@Test
@@ -138,10 +142,13 @@ public class QuickStartIT extends BaseResourceIT {
assertEquals(columns.size(), 4);
- Referenceable column = columns.get(0);
+ Referenceable timeIdColumn = columns.stream()
+ .filter(c -> QuickStart.TIME_ID_COLUMN.equals(c.get("name")))
+ .findFirst()
+ .orElse(null);
- assertEquals(column.get("name"), QuickStart.TIME_ID_COLUMN);
- assertEquals(column.get("dataType"), "int");
+ assertNotNull(timeIdColumn);
+ assertEquals(timeIdColumn.get("dataType"), "int");
}
private void verifyDBIsLinkedToTable(Referenceable table) throws
AtlasServiceException {
diff --git a/webapp/src/test/java/org/apache/atlas/examples/QuickStartV2IT.java
b/webapp/src/test/java/org/apache/atlas/examples/QuickStartV2IT.java
index aaec24dbf..c020ccc2c 100644
--- a/webapp/src/test/java/org/apache/atlas/examples/QuickStartV2IT.java
+++ b/webapp/src/test/java/org/apache/atlas/examples/QuickStartV2IT.java
@@ -36,7 +36,9 @@ import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import java.util.UUID;
+import java.util.stream.Collectors;
import static org.apache.atlas.AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME;
import static org.apache.atlas.examples.QuickStartV2.CLUSTER_SUFFIX;
@@ -119,9 +121,13 @@ public class QuickStartV2IT extends BaseResourceIT {
String timeDimTableId = getTableId(TIME_DIM_TABLE);
String salesFactDailyMVId = getTableId(SALES_FACT_DAILY_MV_TABLE);
- assertEquals(salesFactTableId, ((Map<?, ?>)
inputs.get(0)).get("guid"));
- assertEquals(timeDimTableId, ((Map<?, ?>) inputs.get(1)).get("guid"));
- assertEquals(salesFactDailyMVId, ((Map<?, ?>)
outputs.get(0)).get("guid"));
+ Set<String> inputGuids = inputs.stream()
+ .map(i -> (String) ((Map<?, ?>) i).get("guid"))
+ .collect(Collectors.toSet());
+
+ assertTrue(inputGuids.contains(salesFactTableId));
+ assertTrue(inputGuids.contains(timeDimTableId));
+ assertEquals(((Map<?, ?>) outputs.get(0)).get("guid"),
salesFactDailyMVId);
}
@Test
diff --git
a/webapp/src/test/java/org/apache/atlas/web/integration/BasicSearchIT.java
b/webapp/src/test/java/org/apache/atlas/web/integration/BasicSearchIT.java
index d814c7771..24c7b1924 100644
--- a/webapp/src/test/java/org/apache/atlas/web/integration/BasicSearchIT.java
+++ b/webapp/src/test/java/org/apache/atlas/web/integration/BasicSearchIT.java
@@ -57,6 +57,8 @@ import static org.testng.Assert.assertTrue;
import static org.testng.Assert.fail;
public class BasicSearchIT extends BaseResourceIT {
+ private static final String IMPORTED_DATA_QUALIFIED_NAME_MARKER = "@cl1";
+
private AtlasUserSavedSearch userSavedSearch;
@BeforeClass
@@ -141,14 +143,15 @@ public class BasicSearchIT extends BaseResourceIT {
LOG.info("TestDescription :{}",
testExpectation.testDescription);
LOG.info("SearchParameters :{}",
testExpectation.searchParameters);
- AtlasSearchResult searchResult =
atlasClientV2.facetedSearch(testExpectation.searchParameters);
+ SearchParameters searchParameters =
scopeToImportedDataset(testExpectation.searchParameters);
+ AtlasSearchResult searchResult =
atlasClientV2.facetedSearch(searchParameters);
if (testExpectation.expectedCount > 0) {
assertNotNull(searchResult.getEntities());
assertEquals(searchResult.getEntities().size(),
testExpectation.expectedCount);
}
- if (testExpectation.searchParameters.getSortBy() != null &&
!testExpectation.searchParameters.getSortBy().isEmpty()) {
+ if (searchParameters.getSortBy() != null &&
!searchParameters.getSortBy().isEmpty()) {
assertNotNull(searchResult.getEntities());
assertEquals(searchResult.getEntities().get(0).getAttribute("name"),
"testtable_3");
}
@@ -169,7 +172,7 @@ public class BasicSearchIT extends BaseResourceIT {
LOG.info("TestDescription :{}",
testExpectation.testDescription);
LOG.info("SearchParameters :{}",
testExpectation.searchParameters);
- SearchParameters parameters =
testExpectation.getSearchParameters();
+ SearchParameters parameters =
scopeToImportedDataset(testExpectation.getSearchParameters());
if (parameters.getEntityFilters() == null ||
parameters.getEntityFilters().getAttributeName() == null) {
continue;
@@ -204,7 +207,7 @@ public class BasicSearchIT extends BaseResourceIT {
LOG.info("TestDescription :{}",
testExpectation.testDescription);
LOG.info("SearchParameters :{}",
testExpectation.searchParameters);
- SearchParameters parameters =
testExpectation.getSearchParameters();
+ SearchParameters parameters =
scopeToImportedDataset(testExpectation.getSearchParameters());
AtlasUserSavedSearch savedSearch = new AtlasUserSavedSearch();
@@ -336,6 +339,38 @@ public class BasicSearchIT extends BaseResourceIT {
return containsAnyMatchingCriteria(filter, fc ->
StringUtils.isBlank(fc.getAttributeValue()));
}
+ private SearchParameters scopeToImportedDataset(SearchParameters
parameters) {
+ if (parameters == null || parameters.getTypeName() == null) {
+ return parameters;
+ }
+
+ String typeName = parameters.getTypeName();
+
+ if (!"hive_table".equals(typeName) && !"hive_column".equals(typeName))
{
+ return parameters;
+ }
+
+ SearchParameters.FilterCriteria importedDataFilter = new
SearchParameters.FilterCriteria();
+
+ importedDataFilter.setAttributeName("qualifiedName");
+ importedDataFilter.setOperator(SearchParameters.Operator.CONTAINS);
+
importedDataFilter.setAttributeValue(IMPORTED_DATA_QUALIFIED_NAME_MARKER);
+
+ SearchParameters.FilterCriteria scopedFilter = new
SearchParameters.FilterCriteria();
+
+
scopedFilter.setCondition(SearchParameters.FilterCriteria.Condition.AND);
+
+ if (parameters.getEntityFilters() != null) {
+ scopedFilter.setCriterion(Arrays.asList(importedDataFilter,
parameters.getEntityFilters()));
+ } else {
+
scopedFilter.setCriterion(Collections.singletonList(importedDataFilter));
+ }
+
+ parameters.setEntityFilters(scopedFilter);
+
+ return parameters;
+ }
+
@Test(dependsOnMethods = "testSavedSearch")
public void testExecuteSavedSearchByName() {
try {
