This is an automated email from the ASF dual-hosted git repository. pinal pushed a commit to branch ATLAS-5195 in repository https://gitbox.apache.org/repos/asf/atlas.git
commit 274ce06ad3f6b351dfb43b762afb0fc66bf410b1 Author: Pinal Shah <[email protected]> AuthorDate: Tue Jan 27 12:16:12 2026 +0530 ATLAS-5195: Support to run ITs against postgres backend --- build-tools/it/it-rdbms-start-postgres.sh | 48 ++++++++ build-tools/it/it-rdbms-stop-postgres.sh | 25 ++++ build-tools/it/it-rdbms-wait-atlas-active.sh | 35 ++++++ dev-support/atlas-docker/scripts/atlas-build.sh | 2 + pom.xml | 28 +++++ webapp/pom.xml | 100 +++++++++++++++ .../resources-rdbms/atlas-application.properties | 135 +++++++++++++++++++++ 7 files changed, 373 insertions(+) diff --git a/build-tools/it/it-rdbms-start-postgres.sh b/build-tools/it/it-rdbms-start-postgres.sh new file mode 100644 index 000000000..dafb1c3c7 --- /dev/null +++ b/build-tools/it/it-rdbms-start-postgres.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -euo pipefail + +CONTAINER_NAME="${ATLAS_IT_RDBMS_CONTAINER_NAME:-atlas-it-postgres}" +IMAGE="${ATLAS_IT_RDBMS_IMAGE:-postgres:13.21}" +HOST_PORT="${ATLAS_IT_RDBMS_HOSTPORT:-15432}" +USERNAME="${ATLAS_IT_RDBMS_USERNAME:-atlas}" +PASSWORD="${ATLAS_IT_RDBMS_PASSWORD:-atlasR0cks!}" +DB_NAME="${ATLAS_IT_RDBMS_DBNAME:-atlas}" +MAX_CONNECTIONS="${ATLAS_IT_RDBMS_MAX_CONNECTIONS:-300}" + +docker rm -f "${CONTAINER_NAME}" >/dev/null 2>&1 || true + +docker run -d --name "${CONTAINER_NAME}" \ + -e "POSTGRES_DB=${DB_NAME}" \ + -e "POSTGRES_USER=${USERNAME}" \ + -e "POSTGRES_PASSWORD=${PASSWORD}" \ + -p "${HOST_PORT}:5432" \ + "${IMAGE}" -c "max_connections=${MAX_CONNECTIONS}" >/dev/null + +for i in $(seq 1 60); do + docker exec "${CONTAINER_NAME}" pg_isready -U "${USERNAME}" -d "${DB_NAME}" >/dev/null 2>&1 && exit 0 + sleep 1 +done + +echo "Postgres container did not become ready in time" >&2 +docker logs "${CONTAINER_NAME}" || true +exit 1 + + diff --git a/build-tools/it/it-rdbms-stop-postgres.sh b/build-tools/it/it-rdbms-stop-postgres.sh new file mode 100644 index 000000000..8e70992fa --- /dev/null +++ b/build-tools/it/it-rdbms-stop-postgres.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -euo pipefail + +CONTAINER_NAME="${ATLAS_IT_RDBMS_CONTAINER_NAME:-atlas-it-postgres}" +docker rm -f "${CONTAINER_NAME}" >/dev/null 2>&1 || true + + diff --git a/build-tools/it/it-rdbms-wait-atlas-active.sh b/build-tools/it/it-rdbms-wait-atlas-active.sh new file mode 100644 index 000000000..676a60e40 --- /dev/null +++ b/build-tools/it/it-rdbms-wait-atlas-active.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -euo pipefail + +STATUS_URL="${ATLAS_IT_STATUS_URL:-http://localhost:31000/api/atlas/admin/status}"; +MAX_SECONDS="${ATLAS_IT_WAIT_ACTIVE_SECONDS:-180}" + +for i in $(seq 1 "${MAX_SECONDS}"); do + out="$(curl -fsS "${STATUS_URL}" 2>/dev/null || true)" + echo "${out}" | grep -q ACTIVE && exit 0 + sleep 1 +done + +echo "Atlas did not become ACTIVE in time" >&2 +curl -v "${STATUS_URL}" || true +exit 1 + + diff --git a/dev-support/atlas-docker/scripts/atlas-build.sh b/dev-support/atlas-docker/scripts/atlas-build.sh index ff60538e8..bb21f5685 100755 --- a/dev-support/atlas-docker/scripts/atlas-build.sh +++ b/dev-support/atlas-docker/scripts/atlas-build.sh @@ -87,6 +87,8 @@ else fi mvn ${ARG_PROFILES} ${ARG_SKIPTESTS} -DskipDocs clean package +mvn -Pit-rdbms -DskipUTs=true -DskipTests=false -DskipITs=false verify +mvn -Pdist -DskipUTs=true -DskipTests=false -DskipITs=false verify mv -f distro/target/apache-atlas-${ATLAS_VERSION}-server.tar.gz /home/atlas/dist/ mv -f distro/target/apache-atlas-${ATLAS_VERSION}-hive-hook.tar.gz /home/atlas/dist/ diff --git a/pom.xml b/pom.xml index 9e0a0999a..8f586846d 100644 --- a/pom.xml +++ b/pom.xml @@ -2396,5 +2396,33 @@ </properties> </profile> + <profile> + <id>it-rdbms</id> + <properties> + <!-- Enunciate uses javac internals and fails on newer JDKs without module exports; not needed for IT runs --> + <skipEnunciate>true</skipEnunciate> + <skipITs>false</skipITs> + </properties> + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-failsafe-plugin</artifactId> + <executions> + <execution> + <id>integration-test</id> + <configuration> + <includes> + <include>**/*IT.java</include> + </includes> + <excludes combine.self="override" /> + </configuration> + </execution> + </executions> + </plugin> + </plugins> + </build> + </profile> + </profiles> </project> diff --git a/webapp/pom.xml b/webapp/pom.xml index c314e5857..8bb0f1387 100755 --- a/webapp/pom.xml +++ b/webapp/pom.xml @@ -866,6 +866,106 @@ </properties> </profile> + <!-- + Profile for running Postgres/RDBMS-backed integration tests. + This profile overlays an alternate atlas-application.properties onto target/test-classes + (atlas.conf is already set to ${project.build.directory}/test-classes for Jetty ITs). + --> + <profile> + <id>it-rdbms</id> + <properties> + <it.rdbms.containerName>atlas-it-postgres</it.rdbms.containerName> + <it.rdbms.dbName>atlas</it.rdbms.dbName> + <it.rdbms.docker.skip>false</it.rdbms.docker.skip> + <it.rdbms.hostPort>15432</it.rdbms.hostPort> + <it.rdbms.image>postgres:13.21</it.rdbms.image> + <it.rdbms.jdbcUrl>jdbc:postgresql://localhost:${it.rdbms.hostPort}/atlas</it.rdbms.jdbcUrl> + <it.rdbms.maxConnections>300</it.rdbms.maxConnections> + <it.rdbms.password>itpwd</it.rdbms.password> + <it.rdbms.scriptsDir>${projectBaseDir}/build-tools/it</it.rdbms.scriptsDir> + <it.rdbms.username>atlas</it.rdbms.username> + <it.rdbms.waitActiveSeconds>180</it.rdbms.waitActiveSeconds> + </properties> + <build> + <testResources> + <testResource> + <filtering>true</filtering> + <directory>src/test/resources</directory> + </testResource> + <testResource> + <filtering>true</filtering> + <directory>src/test/resources-rdbms</directory> + </testResource> + </testResources> + <plugins> + <plugin> + <groupId>org.codehaus.mojo</groupId> + <artifactId>exec-maven-plugin</artifactId> + <version>3.5.0</version> + <executions> + <execution> + <id>it-rdbms-start-postgres</id> + <goals> + <goal>exec</goal> + </goals> + <phase>initialize</phase> + <configuration> + <skip>${it.rdbms.docker.skip}</skip> + <executable>bash</executable> + <environmentVariables> + <ATLAS_IT_RDBMS_CONTAINER_NAME>${it.rdbms.containerName}</ATLAS_IT_RDBMS_CONTAINER_NAME> + <ATLAS_IT_RDBMS_IMAGE>${it.rdbms.image}</ATLAS_IT_RDBMS_IMAGE> + <ATLAS_IT_RDBMS_HOSTPORT>${it.rdbms.hostPort}</ATLAS_IT_RDBMS_HOSTPORT> + <ATLAS_IT_RDBMS_USERNAME>${it.rdbms.username}</ATLAS_IT_RDBMS_USERNAME> + <ATLAS_IT_RDBMS_PASSWORD>${it.rdbms.password}</ATLAS_IT_RDBMS_PASSWORD> + <ATLAS_IT_RDBMS_DBNAME>${it.rdbms.dbName}</ATLAS_IT_RDBMS_DBNAME> + <ATLAS_IT_RDBMS_MAX_CONNECTIONS>${it.rdbms.maxConnections}</ATLAS_IT_RDBMS_MAX_CONNECTIONS> + </environmentVariables> + <arguments> + <argument>${it.rdbms.scriptsDir}/it-rdbms-start-postgres.sh</argument> + </arguments> + </configuration> + </execution> + <execution> + <id>it-rdbms-wait-atlas-active</id> + <goals> + <goal>exec</goal> + </goals> + <phase>pre-integration-test</phase> + <configuration> + <executable>bash</executable> + <environmentVariables> + <ATLAS_IT_STATUS_URL>http://localhost:31000/api/atlas/admin/status</ATLAS_IT_STATUS_URL> + <ATLAS_IT_WAIT_ACTIVE_SECONDS>${it.rdbms.waitActiveSeconds}</ATLAS_IT_WAIT_ACTIVE_SECONDS> + </environmentVariables> + <arguments> + <argument>${it.rdbms.scriptsDir}/it-rdbms-wait-atlas-active.sh</argument> + </arguments> + </configuration> + </execution> + <execution> + <id>it-rdbms-stop-postgres</id> + <goals> + <goal>exec</goal> + </goals> + <phase>post-integration-test</phase> + <configuration> + <skip>${it.rdbms.docker.skip}</skip> + <executable>bash</executable> + <environmentVariables> + <ATLAS_IT_RDBMS_CONTAINER_NAME>${it.rdbms.containerName}</ATLAS_IT_RDBMS_CONTAINER_NAME> + </environmentVariables> + <arguments> + <argument>${it.rdbms.scriptsDir}/it-rdbms-stop-postgres.sh</argument> + </arguments> + </configuration> + </execution> + </executions> + </plugin> + </plugins> + </build> + </profile> + <profile> <id>Windows</id> <activation> diff --git a/webapp/src/test/resources-rdbms/atlas-application.properties b/webapp/src/test/resources-rdbms/atlas-application.properties new file mode 100644 index 000000000..6f07c2d1c --- /dev/null +++ b/webapp/src/test/resources-rdbms/atlas-application.properties @@ -0,0 +1,135 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +######### Atlas Server Configs ######### +atlas.rest.address=http://localhost:31000 + +######### Graph Database Configs ######### +atlas.graph.index.search.solr.wait-searcher=true + +# Graph database implementation. Value inserted by maven. +atlas.graphdb.backend=org.apache.atlas.repository.graphdb.janus.AtlasJanusGraphDatabase + +# +# RDBMS (Postgres) backend for JanusGraph +# +atlas.graph.storage.backend=rdbms + +# JDBC/Hikari settings (override these in CI if needed) +atlas.graph.storage.rdbms.jpa.hikari.driverClassName=org.postgresql.Driver +atlas.graph.storage.rdbms.jpa.hikari.jdbcUrl=${it.rdbms.jdbcUrl} +atlas.graph.storage.rdbms.jpa.hikari.username=${it.rdbms.username} +atlas.graph.storage.rdbms.jpa.hikari.password=${it.rdbms.password} +atlas.graph.storage.rdbms.jpa.hikari.maximumPoolSize=40 +atlas.graph.storage.rdbms.jpa.hikari.minimumIdle=5 +atlas.graph.storage.rdbms.jpa.hikari.connectionTestQuery=select 1 +atlas.graph.storage.rdbms.jpa.hikari.connectionTimeout=60000 +atlas.graph.storage.rdbms.jpa.hikari.idleTimeout=300000 +atlas.graph.storage.rdbms.jpa.hikari.maxLifetime=1800000 +atlas.graph.storage.rdbms.jpa.hikari.leakDetectionThreshold=60000 + +# JPA schema init for JanusGraph RDBMS store +atlas.graph.storage.rdbms.jpa.javax.persistence.jdbc.dialect=org.eclipse.persistence.platform.database.PostgreSQLPlatform +atlas.graph.storage.rdbms.jpa.javax.persistence.schema-generation.database.action=create +atlas.graph.storage.rdbms.jpa.javax.persistence.schema-generation.create-database-schemas=true +atlas.graph.storage.rdbms.jpa.javax.persistence.schema-generation.create-source=script +atlas.graph.storage.rdbms.jpa.javax.persistence.schema-generation.create-script-source=META-INF/postgres/create_schema.sql + +# Entity audit repository for ITs (avoid HBase audit dependency) +# NOTE: RdbmsBasedAuditRepository currently doesn't implement V1 audit event writes used by ITs. +atlas.EntityAuditRepository.impl=org.apache.atlas.repository.audit.InMemoryEntityAuditRepository + +# Graph Search Index Backend +atlas.graph.index.search.backend=solr + +# Solr cloud mode properties +atlas.graph.index.search.solr.mode=cloud +atlas.graph.index.search.solr.zookeeper-url=${solr.zk.address} +atlas.graph.index.search.solr.embedded=true +atlas.graph.index.search.max-result-set-size=150 + +######### Notification Configs ######### +atlas.notification.embedded=true +atlas.notification.entity.version=v1 + +atlas.kafka.zookeeper.connect=localhost:19026 +atlas.kafka.bootstrap.servers=localhost:19027 +atlas.kafka.data=${sys:atlas.data}/kafka +atlas.kafka.zookeeper.session.timeout.ms=4000 +atlas.kafka.zookeeper.sync.time.ms=20 +atlas.kafka.consumer.timeout.ms=4000 +atlas.kafka.auto.commit.interval.ms=100 +atlas.kafka.hook.group.id=atlas +atlas.kafka.entities.group.id=atlas_entities + +atlas.kafka.enable.auto.commit=false +atlas.kafka.auto.offset.reset=earliest +atlas.kafka.session.timeout.ms=30000 +atlas.kafka.offsets.topic.replication.factor=1 + +######### Entity Audit Configs ######### +atlas.audit.hbase.tablename=ATLAS_ENTITY_AUDIT_EVENTS +atlas.audit.zookeeper.session.timeout.ms=1000 +atlas.audit.hbase.zookeeper.quorum=localhost +atlas.audit.hbase.zookeeper.property.clientPort=19026 + +######### Security Properties ######### + +# SSL config +atlas.enableTLS=false +atlas.server.https.port=31443 + +######### Security Properties ######### + +hbase.security.authentication=simple + +atlas.hook.falcon.synchronous=true + +######### JAAS Configuration ######## + +atlas.jaas.KafkaClient.loginModuleName = com.sun.security.auth.module.Krb5LoginModule +atlas.jaas.KafkaClient.loginModuleControlFlag = required +atlas.jaas.KafkaClient.option.useKeyTab = true +atlas.jaas.KafkaClient.option.storeKey = true +atlas.jaas.KafkaClient.option.serviceName = kafka +atlas.jaas.KafkaClient.option.keyTab = /etc/security/keytabs/atlas.service.keytab +atlas.jaas.KafkaClient.option.principal = atlas/[email protected] + +######### High Availability Configuration ######## +atlas.server.ha.enabled=false + +######### Atlas Authorization ######### +atlas.authorizer.impl=none + +######### Atlas Authentication ######### +atlas.authentication.method.file=true +atlas.authentication.method.ldap.type=none +atlas.authentication.method.kerberos=false + +######### Gremlin Search Configuration ######### +# Set to false to disable gremlin search. +atlas.search.gremlin.enable=true + +######### Configure use of Tasks ######### +atlas.tasks.enabled=false +atlas.debug.metrics.enabled=true + +######### Configure on-demand lineage ######### +atlas.lineage.on.demand.enabled=true + +
